From e00da50e625ba2f1f3cecfa11cfc916ddae7e8d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Courvoisier?= Date: Tue, 14 Jan 2025 14:09:56 +0100 Subject: [PATCH] We forgot to start the normalization batch --- .gitlab-ci.yml | 27 +++++++++++- Dockerfile | 20 ++++++++- .../defaults/main/defaults.yml | 3 +- .../tasks/deployment/deployment_batch.yml | 42 +++++++++++++++++++ batch_docker_entrypoint.sh | 4 ++ docker-compose.local.yml | 15 +++++++ nodemon-batch.json | 5 +++ package.json | 3 ++ 8 files changed, 114 insertions(+), 5 deletions(-) create mode 100644 ansible/roles/deploy_juritcom/tasks/deployment/deployment_batch.yml create mode 100644 batch_docker_entrypoint.sh create mode 100644 nodemon-batch.json diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3f1211c..609295d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,8 @@ variables: re7: preprod master: prod api_docker_image: '$CI_REGISTRY/cour-de-cassation/juritcom/api:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA' - + batch_docker_image: '$CI_REGISTRY/cour-de-cassation/juritcom/batch:$CI_COMMIT_BRANCH-$CI_COMMIT_SHORT_SHA' + stages: - test - build @@ -60,6 +61,27 @@ build_api: tags: - docker +build_batch: + stage: build +# needs: ['test'] + variables: + HTTP_PROXY: $HTTP_PROXY_DEV + HTTPS_PROXY: $HTTPS_PROXY_DEV + script: + - echo $CI_JOB_TOKEN | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY + - docker build + --build-arg http_proxy=$HTTP_PROXY + --build-arg https_proxy=$HTTPS_PROXY + --target batch + -t $batch_docker_image . + - docker push $batch_docker_image + only: + - master + - re7 + - dev + tags: + - docker + deploy_juritcom: stage: deploy image: alpine/ansible:2.16.1 @@ -72,7 +94,7 @@ deploy_juritcom: - cat $KNOWN_HOSTS > /root/.ssh/known_hosts - chmod 600 /root/.ssh/id_rsa - chmod 600 /root/.ssh/known_hosts - - ansible-playbook -e juritcom_api_image=$api_docker_image -i ansible/inventory/$inventaire.yml ansible/deploy_juritcom.yml --vault-password-file=$ANSIBLE_VAULT_PASS + - ansible-playbook -e juritcom_api_image=$api_docker_image -e juritcom_batch_image=$batch_docker_image -i ansible/inventory/$inventaire.yml ansible/deploy_juritcom.yml --vault-password-file=$ANSIBLE_VAULT_PASS rules: - if: $CI_COMMIT_BRANCH == "master" when: manual @@ -82,3 +104,4 @@ deploy_juritcom: - docker needs: - build_api + - build_batch diff --git a/Dockerfile b/Dockerfile index bfcf515..2bbb435 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,16 @@ WORKDIR /home/node COPY --from=prod --chown=node:node /home/node/package*.json ./ COPY --from=prod --chown=node:node /home/node/node_modules/ ./node_modules/ +COPY --from=prod --chown=node:node /home/node/dist/shared ./dist/shared + +# --- Base final image with batch dist content --- # +FROM shared as batch + +USER node +COPY --from=prod --chown=node:node /home/node/dist/batch ./dist/batch +COPY --chown=node:node batch_docker_entrypoint.sh batch_docker_entrypoint.sh + +ENTRYPOINT ["/bin/sh", "batch_docker_entrypoint.sh"] # --- Base final image with api dist content --- # FROM shared as api @@ -47,8 +57,15 @@ COPY --from=prod --chown=node:node /home/node/dist ./dist CMD ["node", "dist/api/main"] +# --- Base image with batch content --- # +FROM shared as batch-local + +USER node + +CMD ["npm", "run", "batch:start:watch"] + # --- Base image with api content --- # -FROM node:18-alpine as api-local +FROM node:20-alpine as api-local USER node WORKDIR /home/node @@ -57,4 +74,3 @@ COPY --chown=node:node . . RUN npm i CMD ["npm", "run", "start:dev"] - diff --git a/ansible/roles/deploy_juritcom/defaults/main/defaults.yml b/ansible/roles/deploy_juritcom/defaults/main/defaults.yml index ab66f76..04bb4d8 100644 --- a/ansible/roles/deploy_juritcom/defaults/main/defaults.yml +++ b/ansible/roles/deploy_juritcom/defaults/main/defaults.yml @@ -2,4 +2,5 @@ juritcom_namespace: juritcom juritcom_application: juritcom -api_app_id: api \ No newline at end of file +api_app_id: api +batch_app_id: batch \ No newline at end of file diff --git a/ansible/roles/deploy_juritcom/tasks/deployment/deployment_batch.yml b/ansible/roles/deploy_juritcom/tasks/deployment/deployment_batch.yml new file mode 100644 index 0000000..6cf187c --- /dev/null +++ b/ansible/roles/deploy_juritcom/tasks/deployment/deployment_batch.yml @@ -0,0 +1,42 @@ +--- +- name: Create batch Deployment + k8s: + apply: true + state: present + verify_ssl: true + definition: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: "{{ batch_app_id }}-deployment" + namespace: "{{ juritcom_namespace }}" + labels: + app: + spec: + replicas: 1 + selector: + matchLabels: + app: "{{ batch_app_id }}" + template: + metadata: + labels: + app: "{{ batch_app_id }}" + spec: + containers: + - name: "{{ batch_app_id }}" + image: "{{ juritcom_batch_image }}" + env: + - name: NORMALIZATION_BATCH_SCHEDULE + value: "{{ normalization_batch_schedule }}" + - name: COMMISSIONING_DATE + value: "{{ commissioning_date }}" + envFrom: + - configMapRef: + name: "api-config" + - secretRef: + name: "api-secret" + - configMapRef: + name: "s3-config" + - secretRef: + name: "s3-secret" +... diff --git a/batch_docker_entrypoint.sh b/batch_docker_entrypoint.sh new file mode 100644 index 0000000..4b07291 --- /dev/null +++ b/batch_docker_entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -e + +npm run batch:start:prod \ No newline at end of file diff --git a/docker-compose.local.yml b/docker-compose.local.yml index db51852..3becad4 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -39,6 +39,21 @@ services: ports: - 3009:3000 + batch: + build: + context: ./ + target: batch + env_file: + - docker.env + networks: + - default + - dbsder-external-network + +networks: + default: + dbsder-external-network: + external: true + volumes: bucket-storage: driver: local diff --git a/nodemon-batch.json b/nodemon-batch.json new file mode 100644 index 0000000..217e727 --- /dev/null +++ b/nodemon-batch.json @@ -0,0 +1,5 @@ +{ + "watch": ["src/shared", "src/batch"], + "ext": "ts", + "exec": "ts-node -r dotenv/config src/batch/normalization/index.ts" +} \ No newline at end of file diff --git a/package.json b/package.json index af388c2..294bcbd 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,9 @@ "start:dev": "NODE_ENV=local nest start --watch", "start:debug": "nest start --debug --watch", "start:prod": "node dist/api/main", + "batch:start": "npx ts-node -r dotenv/config src/batch/normalization/index.ts", + "batch:start:watch": "nodemon --config nodemon-batch.json", + "batch:start:prod": "node -r dotenv/config dist/batch/normalization/index.js", "docker:build": "docker compose -f docker-compose.local.yml build", "docker:start": "docker compose -f docker-compose.local.yml up -d", "docker:stop": "docker compose -f docker-compose.local.yml stop",