Merge pull request #37 from Cour-de-cassation/dev

Start normalization batch

.gitlab-ci.yml

@@ -7,7 +7,8 @@ variables:
   re7: preprod
   master: prod
   api_docker_image: '$CI_REGISTRY/cour-de-cassation/juritcom/api:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA'
-
+  batch_docker_image: '$CI_REGISTRY/cour-de-cassation/juritcom/batch:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA'
+
 stages:
   - test
   - build
@@ -60,6 +61,27 @@ build_api:
   tags:
     - docker
 
+build_batch:
+  stage: build
+#  needs: ['test']
+  variables:
+    HTTP_PROXY: $HTTP_PROXY_DEV
+    HTTPS_PROXY: $HTTPS_PROXY_DEV
+  script:
+    - echo $CI_JOB_TOKEN | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
+    - docker build
+      --build-arg http_proxy=$HTTP_PROXY
+      --build-arg https_proxy=$HTTPS_PROXY
+      --target batch
+      -t $batch_docker_image .
+    - docker push $batch_docker_image
+  only:
+    - master
+    - re7
+    - dev
+  tags:
+    - docker
+
 deploy_juritcom:
   stage: deploy
   image: alpine/ansible:2.16.1
@@ -72,7 +94,7 @@ deploy_juritcom:
     - cat $KNOWN_HOSTS > /root/.ssh/known_hosts
     - chmod 600 /root/.ssh/id_rsa
     - chmod 600 /root/.ssh/known_hosts
-    - ansible-playbook -e juritcom_api_image=$api_docker_image -i ansible/inventory/$inventaire.yml ansible/deploy_juritcom.yml --vault-password-file=$ANSIBLE_VAULT_PASS
+    - ansible-playbook -e juritcom_api_image=$api_docker_image -e juritcom_batch_image=$batch_docker_image -i ansible/inventory/$inventaire.yml ansible/deploy_juritcom.yml --vault-password-file=$ANSIBLE_VAULT_PASS
   rules:
     - if: $CI_COMMIT_BRANCH == "master"
       when: manual
@@ -82,3 +104,4 @@ deploy_juritcom:
     - docker
   needs:
     - build_api
+    - build_batch

Dockerfile

@@ -36,6 +36,16 @@ WORKDIR /home/node
 
 COPY --from=prod --chown=node:node /home/node/package*.json ./
 COPY --from=prod --chown=node:node /home/node/node_modules/ ./node_modules/
+COPY --from=prod --chown=node:node /home/node/dist/shared ./dist/shared
+
+# --- Base final image with batch dist content --- #
+FROM shared as batch 
+
+USER node
+COPY --from=prod --chown=node:node /home/node/dist/batch ./dist/batch
+COPY --chown=node:node batch_docker_entrypoint.sh batch_docker_entrypoint.sh
+
+ENTRYPOINT ["/bin/sh", "batch_docker_entrypoint.sh"]
 
 # --- Base final image with api dist content --- #
 FROM shared as api
@@ -47,8 +57,15 @@ COPY --from=prod --chown=node:node /home/node/dist ./dist
 
 CMD ["node", "dist/api/main"]
 
+# --- Base image with batch content --- #
+FROM shared as batch-local
+
+USER node
+
+CMD ["npm", "run", "batch:start:watch"]
+
 # --- Base image with api content --- #
-FROM node:18-alpine as api-local
+FROM node:20-alpine as api-local
 
 USER node
 WORKDIR /home/node
@@ -57,4 +74,3 @@ COPY --chown=node:node . .
 RUN npm i
 
 CMD ["npm", "run", "start:dev"]
-

ansible/roles/deploy_juritcom/defaults/main/defaults.yml

@@ -2,4 +2,5 @@
 juritcom_namespace: juritcom
 juritcom_application: juritcom
 
-api_app_id: api
+api_app_id: api
+batch_app_id: batch

ansible/roles/deploy_juritcom/tasks/deployment/deployment_batch.yml

@@ -0,0 +1,42 @@
+---
+- name: Create batch Deployment
+  k8s:
+    apply: true
+    state: present
+    verify_ssl: true
+    definition:
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: "{{ batch_app_id }}-deployment"
+        namespace: "{{ juritcom_namespace }}"
+        labels:
+          app:
+      spec:
+        replicas: 1
+        selector:
+          matchLabels:
+            app: "{{ batch_app_id }}"
+        template:
+          metadata:
+            labels:
+              app: "{{ batch_app_id }}"
+          spec:
+            containers:
+              - name: "{{ batch_app_id }}"
+                image: "{{ juritcom_batch_image }}"
+                env:
+                  - name: NORMALIZATION_BATCH_SCHEDULE
+                    value: "{{ normalization_batch_schedule }}"
+                  - name: COMMISSIONING_DATE
+                    value: "{{ commissioning_date }}"
+                envFrom:
+                  - configMapRef:
+                      name: "api-config"
+                  - secretRef:
+                      name: "api-secret"
+                  - configMapRef:
+                      name: "s3-config"
+                  - secretRef:
+                      name: "s3-secret"
+...

batch_docker_entrypoint.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+set -e 
+
+npm run batch:start:prod

docker-compose.local.yml

@@ -39,6 +39,21 @@ services:
     ports:
       - 3009:3000
 
+  batch: 
+    build:
+      context: ./
+      target: batch
+    env_file:
+      - docker.env
+    networks:
+      - default
+      - dbsder-external-network
+
+networks:
+  default:
+  dbsder-external-network:
+    external: true
+
 volumes:
   bucket-storage:
     driver: local

nodemon-batch.json

@@ -0,0 +1,5 @@
+{
+    "watch": ["src/shared", "src/batch"],
+    "ext": "ts",
+    "exec": "ts-node -r dotenv/config src/batch/normalization/index.ts"
+}

package.json

@@ -13,6 +13,9 @@
     "start:dev": "NODE_ENV=local nest start --watch",
     "start:debug": "nest start --debug --watch",
     "start:prod": "node dist/api/main",
+    "batch:start": "npx ts-node -r dotenv/config src/batch/normalization/index.ts",
+    "batch:start:watch": "nodemon --config nodemon-batch.json",
+    "batch:start:prod": "node -r dotenv/config dist/batch/normalization/index.js",
     "docker:build": "docker compose -f docker-compose.local.yml build",
     "docker:start": "docker compose -f docker-compose.local.yml up -d",
     "docker:stop": "docker compose -f docker-compose.local.yml stop",