Closed #848 Enviroment Variables (#855)

linea-it · Feb 19, 2024 · cc5e405 · cc5e405
1 parent 32846a0
commit cc5e405
Show file tree

Hide file tree

Showing 11 changed files with 93 additions and 75 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -24,4 +24,5 @@ repos:
     hooks:
     -   id: isort
         name: isort (python)
+        args: ["--profile", "black"]
         exclude: ^(migrations/|pipelines/|backend/old_apps/)
diff --git a/docker-compose-development-template.yml b/docker-compose-development-template.yml
@@ -108,14 +108,15 @@ services:
       - ./predict_occultation:/app
       - ./archive/predict_occultation:/app/outputs/predict_occultation
     environment:
+      - PARSL_ENV=local # PARSL_ENV define onde será o processamento: local ou linea (Slurm)
       # Acesso ao DB de Catalogo e Administrativo, caso localhost não funcione utilizar o host.docker.internal.
       # Usando Localhost
       - DB_URI=postgresql+psycopg2://untrustedprod:untrusted@localhost:3307/prod_gavo
       - DB_URI_ADMIN=postgresql+psycopg2://postgres:postgres@localhost:5432/postgres
       # # Usando host.docker.internal
       # - DB_URI=postgresql+psycopg2://untrustedprod:untrusted@host.docker.internal:3307/prod_gavo
       # - DB_URI_ADMIN=postgresql+psycopg2://postgres:postgres@host.docker.internal:5432/postgres
-      - PARSL_ENV=local # PARSL_ENV define onde será o processamento: local ou linea (Slurm)
+      - PREDICT_OUTPUTS=/app/outputs/predict_occultation
 
 volumes:
   rabbitmq-data:

diff --git a/docker-compose-production-template.yml b/docker-compose-production-template.yml
@@ -76,30 +76,23 @@ services:
       - rabbit
       - celery_flower
 
-  pipelines:
+  predict_occultation:
     user: "31670:15010"
     group_add:
-        - "900"
-    image: linea/tno:pipelines_f2c9b54
+      - "900"
+    image: linea/tno:predict_occ_32846a0
     restart: always
-    command: /app/daemon.sh
+    command: /app/src/daemon.sh
     network_mode: "host"
     volumes:
-      - /lustre/t1/scratch/users/app.tno/.ssh/:/home/apptno/.ssh
-      - /lustre/t1/scratch/users/app.tno/tno_testing/outputs/runinfo/:/app/runinfo
+      - /lustre/t1/scratch/users/app.tno/.ssh/id_rsa:/home/apptno/.ssh/id_rsa
       - /lustre/t1/scratch/users/app.tno/tno_testing/:/lustre/t1/scratch/users/app.tno/tno_testing
     environment:
-      - DB_URI=postgresql+psycopg2://untrustedprod:untrusted@desdb4.linea.gov.br:5432/prod_gavo
-      - DB_URI_ADMIN=postgresql+psycopg2://tnouseradmindev:TnoDev405060@desdb4.linea.gov.br:5432/tnodbdev
-      - USER=apptno
       - PARSL_ENV=linea
-      - PREFIX_ENV=testing
-      - PIPELINE_PREDIC_OCC=/lustre/t1/scratch/users/app.tno/tno_testing/predict_occ
-      - PIPELINE_PATH=/lustre/t1/scratch/users/app.tno/tno_testing/predict_occ/pipeline
-      - PREDICT_OUTPUTS=/lustre/t1/scratch/users/app.tno/tno_testing/outputs/predict_occ
-      - SLURM_REMOTE_DIR=/lustre/t1/scratch/users/app.tno/tno_testing/configs
-      - PIPE_BIN=/lustre/t1/scratch/users/app.tno/tno_testing/bin
-      - CONDA_ENVS=/lustre/t1/scratch/users/app.tno/tno_testing/miniconda/envs
+      - REMOTE_PIPELINE_ROOT=/lustre/t1/scratch/users/app.tno/tno_testing
+      - PREDICT_OUTPUTS=/lustre/t1/scratch/users/app.tno/tno_testing/outputs/predict_occultation
+      - DB_URI=postgresql+psycopg2://<USERNAME>:<PASSWORD>@desdb4.linea.gov.br:5432/prod_gavo
+      - DB_URI_ADMIN=postgresql+psycopg2://<USERNAME>:<PASSWORD>@desdb4.linea.gov.br:5432/tnodbdev
       - SSHKEY=/home/apptno/.ssh/id_rsa
 
 volumes:

diff --git a/predict_occultation/.dockerignore b/predict_occultation/.dockerignore
@@ -4,3 +4,4 @@ __pycache__/
 *$py.class
 *.pyc
 *.pyo
+*/outputs/
diff --git a/predict_occultation/.gitignore b/predict_occultation/.gitignore
@@ -12,6 +12,9 @@ naif0012.tls
 # Parsl
 /src/runinfo/
 
+# Outputs
+outputs/
+
 # Distribution / packaging
 .Python
 build/

diff --git a/predict_occultation/Dockerfile b/predict_occultation/Dockerfile
@@ -123,27 +123,18 @@ RUN groupadd --gid ${USERGID} ${USERNAME} \
 ENV PIPELINE_ROOT=${APP_HOME}/src
 ENV PIPELINE_PREDIC_OCC=${PIPELINE_ROOT}/predict_occultation
 ENV PIPELINE_PATH=${PIPELINE_PREDIC_OCC}/pipeline
-ENV PREDICT_OUTPUTS=${APP_HOME}/outputs/predict_occultation
 
 # quando parsl_env = local o diretório remoto é o mesmo que o local.
 # mas quando parsl_env = linea o diretório remoto é por ex: /lustre/t1/scratch/users/app.tno/tno_testing
-ENV REMOTE_PIPELINE_ROOT=$PIPELINE_ROOT
-
+# ENV REMOTE_PIPELINE_ROOT=$PIPELINE_ROOT
+# ENV PREDICT_OUTPUTS=${PIPELINE_ROOT}/outputs/predict_occultation
+ENV REMOTE_CONDA_PATH=/opt/conda/bin
 
 COPY --chown=${USERNAME}:ton --chmod=775 src /app/src
 COPY --chown=${USERNAME}:ton --chmod=775 entrypoint.sh ${APP_HOME}
+COPY --chown=${USERNAME}:ton --chmod=775 check_enviroment.py ${APP_HOME}
 
-# COPY --chown=${USERNAME}:ton --chmod=775 src ${PIPELINE_ROOT}
-# COPY --chown=${USERNAME}:ton --chmod=775 src/predict_occultation ${PIPELINE_PREDIC_OCC}
-# COPY --chown=${USERNAME}:ton --chmod=775 src ${PIPELINE_ROOT}
-# COPY --chown=${USERNAME}:ton --chmod=775 predict_occultation ${PIPELINE_PREDIC_OCC}
-# COPY --chown=${USERNAME}:ton --chmod=775 rerun.py ${APP_HOME}
-# COPY --chown=${USERNAME}:ton --chmod=775 rerun.sh ${APP_HOME}
-# COPY --chown=${USERNAME}:ton --chmod=775 daemon.sh ${APP_HOME}
-# COPY --chown=${USERNAME}:ton --chmod=775 entrypoint.sh ${APP_HOME}
-# COPY --chown=${USERNAME}:ton --chmod=775 run_daemon.py ${APP_HOME}
-
-RUN chmod =2775 ${PIPELINE_ROOT} \
+RUN chmod -R =2775 ${PIPELINE_ROOT} \
     && chmod =2775 ${APP_HOME} \
     && mv de440.bsp ${PIPELINE_PATH}/$BSP_PLANETARY_NAME \
     && mv naif0012.tls ${PIPELINE_PATH}/$LEAP_SECOND_NAME

diff --git a/predict_occultation/check_enviroment.py b/predict_occultation/check_enviroment.py
@@ -0,0 +1,39 @@
+import os
+import sys
+
+# Principais variaveis de ambiente
+# - PARSL_ENV=linea
+# - REMOTE_PIPELINE_ROOT=/lustre/t1/scratch/users/app.tno/tno_testing
+# - PIPELINE_PATH=/lustre/t1/scratch/users/app.tno/tno_testing/predict_occultation/pipeline
+# - PREDICT_OUTPUTS=/lustre/t1/scratch/users/app.tno/tno_testing/outputs/predict_occultation
+# - DB_URI=
+# - DB_URI_ADMIN=
+# - SSHKEY=/home/apptno/.ssh/id_rsa
+
+#  Variaveis obrigatórias independente do ambiente
+if "PARSL_ENV" not in os.environ:
+    sys.exit("It is necessary to configure the PARSL_ENV in docker-compose.yml")
+
+if "DB_URI" not in os.environ:
+    sys.exit("It is necessary to configure the DB_URI in docker-compose.yml")
+
+if "DB_URI_ADMIN" not in os.environ:
+    sys.exit("It is necessary to configure the DB_URI_ADMIN in docker-compose.yml")
+
+if "PREDICT_OUTPUTS" not in os.environ:
+    sys.exit("It is necessary to configure the PREDICT_OUTPUTS in docker-compose.yml")
+
+parsl_env = os.getenv("PARSL_ENV")
+print(f"PARSL_ENV: {parsl_env}")
+
+if parsl_env == "linea":
+    # Variaveis obrigatórias para uso no Ambiente Linea.
+    if "REMOTE_PIPELINE_ROOT" not in os.environ:
+        sys.exit(
+            f"It is necessary to configure the REMOTE_PIPELINE_ROOT in docker-compose.yml"
+        )
+
+    if not "SSHKEY" in os.environ:
+        sys.exit(
+            "It is necessary to configure the SSHKEY in docker-compose.yml when PARSL_ENV=linea"
+        )
diff --git a/predict_occultation/entrypoint.sh b/predict_occultation/entrypoint.sh
@@ -1,8 +1,12 @@
 #!/bin/bash --login
+set -o errexit
 umask ug=rwx,o=r
 
+# Verifica as variaveis de ambiente
+/opt/conda/bin/python3 /app/check_enviroment.py
+
 echo "Running Rsync: ${PIPELINE_PREDIC_OCC}"
-rsync -r /app/src/predict_occultation/ ${REMOTE_PIPELINE_ROOT}/predict_occultation
+rsync -r /app/src/predict_occultation/ ${PIPELINE_PREDIC_OCC}/ --exclude outputs/
 
 
 # Baixa os arquivos bsp planetary e leap_second caso não existam.

diff --git a/predict_occultation/src/env.sh b/predict_occultation/src/env.sh
@@ -3,12 +3,24 @@
 echo "Activating py3 environment"
 conda activate py3
 
-echo "Setup PYTHONPATH"
-export PYTHONPATH=${PYTHONPATH}:${PIPELINE_ROOT}:${PIPELINE_PREDIC_OCC}:${PIPELINE_PATH}
 
 if [[ "$PARSL_ENV" = "linea" ]]
 then
+    export PIPELINE_PREDIC_OCC=${REMOTE_PIPELINE_ROOT}/predict_occultation
+    export PIPELINE_PATH=${PIPELINE_PREDIC_OCC}/pipeline
+    export REMOTE_CONDA_PATH=${REMOTE_PIPELINE_ROOT}/miniconda/bin
+
     ulimit -s 100000
     ulimit -u 100000
 fi
+
+echo "REMOTE_PIPELINE_ROOT : ${PIPELINE_ROOT}"
+echo "PIPELINE_PREDIC_OCC  : ${PIPELINE_PREDIC_OCC}"
+echo "PIPELINE_PATH        : ${PIPELINE_PATH}"
+echo "PREDICT_OUTPUTS      : ${PREDICT_OUTPUTS}"
+echo "REMOTE_CONDA_PATH    : ${REMOTE_CONDA_PATH}"
+
+export PYTHONPATH=${PYTHONPATH}:${PIPELINE_ROOT}:${PIPELINE_PREDIC_OCC}:${PIPELINE_PATH}
+echo "PYTHONPATH           ${PYTHONPATH}"
+
 umask 0002
diff --git a/predict_occultation/src/parsl_config.py b/predict_occultation/src/parsl_config.py
@@ -24,40 +24,17 @@ def get_config(key, jobpath):
 
     if parsl_env == "linea":
         # Check Linea PATH envs
-        pipeline_root = os.getenv("REMOTE_PIPELINE_ROOT", None)
-        if not pipeline_root:
-            raise Exception(
-                "It is necessary to configure the REMOTE_PIPELINE_ROOT variable when PARSL_ENV=linea"
-            )
-
-        pipeline_root = Path(pipeline_root)
-        condapath = pipeline_root.joinpath("miniconda/bin")
-        pipeline_pred_occ = pipeline_root.joinpath("predict_occultation")
-        pipeline_path = pipeline_root.joinpath("predict_occultation/pipeline")
-
+        pipeline_root = Path(os.getenv("REMOTE_PIPELINE_ROOT"))
+        condapath = Path(os.getenv("REMOTE_CONDA_PATH"))
+        pipeline_pred_occ = Path(os.getenv("PIPELINE_PREDIC_OCC"))
+        pipeline_path = Path(os.getenv("PIPELINE_PATH"))
         # Diretório de Outputs
-        predict_outputs = os.getenv("PREDICT_OUTPUTS", None)
-        if not predict_outputs:
-            raise Exception(
-                "It is necessary to configure the PREDICT_OUTPUTS variable when PARSL_ENV=linea"
-            )
-        predict_outputs = Path(predict_outputs)
+        predict_outputs = Path(os.getenv("PREDICT_OUTPUTS"))
         predict_outputs.mkdir(parents=True, exist_ok=True)
-
-        #  Linea SSH user keys
-        # sshkey = os.getenv("SSHKEY", "~/.ssh/id_rsa")
-        sshkey = os.getenv("SSHKEY", None)
-        if not sshkey:
-            raise Exception(
-                "It is necessary to configure the SSHKEY variable when PARSL_ENV=linea"
-            )
-
+        # Linea SSH user keys
+        sshkey = os.getenv("SSHKEY")
         # Linea DB prod_gavo DB uri. Catalog DB.
-        db_uri = os.getenv("DB_URI", None)
-        if not db_uri:
-            raise Exception(
-                "It is necessary to configure the DB_URI variable when PARSL_ENV=linea"
-            )
+        db_uri = os.getenv("DB_URI")
 
         # Env.sh que sera executado antes de iniciar as tasks no cluster
         cluster_env_sh = pipeline_pred_occ.joinpath("cluster.sh")
@@ -79,7 +56,7 @@ def get_config(key, jobpath):
                     min_blocks=3,
                     max_blocks=12,
                     parallelism=1,
-                    walltime="15:00:00",
+                    walltime="24:00:00",
                     worker_init=f"source {cluster_env_sh}\n",
                     channel=SSHChannel(
                         hostname="loginapl01",

diff --git a/predict_occultation/src/run_pred_occ.py b/predict_occultation/src/run_pred_occ.py
@@ -22,6 +22,7 @@
 
 try:
     from parsl_config import get_config
+
     from predict_occultation.app import run_pipeline
 except Exception as error:
     print("Error: %s" % str(error))
@@ -174,21 +175,17 @@ def make_job_json_file(job, path):
             "debug": bool(job.get("debug", False)),
             "error": None,
             "traceback": None,
-            # Parsl init block não está sendo utilizado no pipeline
-            # "parsl_init_block": int(job.get('parsl_init_block', 600)),
             # TODO: Adicionar parametro para catalog
             # "catalog_id": job.get('catalog_id')
             # TODO: Estes parametros devem ser gerados pelo pipeline lendo do config.
             # TODO: Bsp e leap second deve fazer a query e verificar o arquivo ou fazer o download.
             "bsp_planetary": {
                 "name": "de440",
                 "filename": "de440.bsp",
-                # "absolute_path": "/lustre/t1/tmp/tno/bsp_planetary/de440.bsp",
             },
             "leap_seconds": {
                 "name": "naif0012",
                 "filename": "naif0012.tls",
-                # "absolute_path": "/lustre/t1/tmp/tno/leap_seconds/naif0012.tls",
             },
             "force_refresh_inputs": True,
             "inputs_days_to_expire": 0,
@@ -223,14 +220,14 @@ def remove_job_directory(jobid):
 
 def get_job_path(jobid):
     """Retorna o path para o diretorio do job, cria o diretorio caso nao exista."""
-    orbit_trace_root = os.getenv("PREDICT_OUTPUTS", "/app/outputs/predict_occultation")
+    predict_outputs = os.getenv("PREDICT_OUTPUTS")
+    print("Predict_outputs: ")
+    print(predict_outputs)
     folder_name = f"{jobid}"
     # folder_name = f"teste_{job['id']}"
     # folder_name = f"{job['id']}-{str(uuid.uuid4())[:8]}"
 
-    # prefix_env = os.getenv('PREFIX_ENV', 'dev')
-    # job_path = pathlib.Path(orbit_trace_root).joinpath(prefix_env, folder_name)
-    job_path = pathlib.Path(orbit_trace_root).joinpath(folder_name)
+    job_path = pathlib.Path(predict_outputs).joinpath(folder_name)
 
     if not job_path.exists():
         job_path.mkdir(parents=True, mode=0o775)
@@ -239,7 +236,6 @@ def get_job_path(jobid):
         script_dir = job_path.joinpath("script_dir")
         script_dir.mkdir(parents=True, mode=0o775)
 
-    print(f"Job Path: {job_path}")
     return job_path