Predict to py3 (#817)

* All information available in the asteroid table was added to the prediction pipeline. #807 * Minor fixes * Closed #807 - Prediction pipeline now records all asteroid data for each prediction event. * Updated documentation * Added progress bar dowload external inputs * Added progress bar to update_asteroid_table command #807 * Recreate container for predict_occultation pipeline. in this moment pipeline running in local success * prediction pipeline has been updated to run in the python 3.8 environment. * Removed files related with orbit Trace #811 * New folder for Orbit Trace Pipeline * Added path coef to pipelie stage * Pipeline predict Running with py3.8 And Path Coeff running inside pipeline. Closed #812 * Update daemon.sh * Refactoring da classe dao. agora separada por arquivos para facilitar a manutenção * minor fixes
linea-it · Feb 6, 2024 · bd268d0 · bd268d0
1 parent d28fe6c
commit bd268d0
Show file tree

Hide file tree

Showing 136 changed files with 32,516 additions and 36 deletions.
diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
@@ -95,6 +95,49 @@ jobs:
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
 
+  build_predict_occultation:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    -
+      name: Checkout
+      uses: actions/checkout@v3
+    -
+      name: Docker meta  # https://github.com/docker/metadata-action
+      id: meta
+      uses: docker/metadata-action@v5
+      with:
+        images: |
+          linea/tno
+        tags: |
+          type=semver,pattern={{version}},prefix=predict_occ_
+          type=sha,format=short,prefix=predict_occ_
+        flavor: |
+          latest=auto
+          prefix=predict_occ_,onlatest=true
+    -
+      name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+    -
+      name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+    -
+      name: Login to Docker Hub
+      if: github.event_name != 'pull_request'
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME  }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    -
+      name: Build and push
+      uses: docker/build-push-action@v5
+      with:
+        context: ./predict_occultation
+        push: ${{ github.event_name != 'pull_request' }}
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}                
+
   build_pipelines:
 
     runs-on: ubuntu-latest

diff --git a/.gitignore b/.gitignore
@@ -58,3 +58,6 @@ env_tno
 core_admin/.astropy
 
 database_subset/
+
+# Parsl Runinfo
+predict_occultation/runinfo
diff --git a/backend/tno/migrations/0059_auto_20240205_1916.py b/backend/tno/migrations/0059_auto_20240205_1916.py
@@ -0,0 +1,28 @@
+# Generated by Django 3.2.18 on 2024-02-05 19:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('tno', '0058_auto_20240201_1703'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='predictionjobresult',
+            name='calc_path_coeff_exec_time',
+            field=models.DurationField(blank=True, help_text='Execution time of the path coeff step.', null=True, verbose_name='Calc Path Coeff Execution Time'),
+        ),
+        migrations.AddField(
+            model_name='predictionjobresult',
+            name='calc_path_coeff_finish',
+            field=models.DateTimeField(blank=True, help_text='End of path coeff.', null=True, verbose_name='Calc Path Coeff Finish'),
+        ),
+        migrations.AddField(
+            model_name='predictionjobresult',
+            name='calc_path_coeff_start',
+            field=models.DateTimeField(blank=True, help_text='Start of path coeff.', null=True, verbose_name='Calc Path Coeff Start'),
+        ),
+    ]
diff --git a/backend/tno/migrations/0060_auto_20240206_1428.py b/backend/tno/migrations/0060_auto_20240206_1428.py
diff --git a/backend/tno/models/occultation.py b/backend/tno/models/occultation.py
@@ -100,31 +100,31 @@ class Occultation(models.Model):
         max_length=20,
         null=False,
         blank=False,
-        help="Star candidate right ascension (hh:mm:ss)",
+        help_text="Star candidate right ascension (hh:mm:ss)",
     )
 
     dec_star_candidate = models.CharField(
         verbose_name="Dec Star Candidate",
         max_length=20,
         null=False,
         blank=False,
-        help="Star candidate declination (dd:mm:ss)",
+        help_text="Star candidate declination (dd:mm:ss)",
     )
 
     ra_target = models.CharField(
         verbose_name="RA Target",
         max_length=20,
         null=False,
         blank=False,
-        help="Target's right ascension (hh:mm:ss)",
+        help_text="Target's right ascension (hh:mm:ss)",
     )
 
     dec_target = models.CharField(
         verbose_name="Dec Target",
         max_length=20,
         null=False,
         blank=False,
-        help="Target's declination (dd:mm:ss)",
+        help_text="Target's declination (dd:mm:ss)",
     )
 
     closest_approach = models.FloatField(

diff --git a/backend/tno/models/prediction_job_result.py b/backend/tno/models/prediction_job_result.py
@@ -240,6 +240,31 @@ class PredictionJobResult(models.Model):
         help_text="Predict Occultation runtime."
     )
 
+    # Etapa de Calculo do Path Coeff, é executado junto com a predição,
+    # Mas por ser um pouco demorado é interessante ter o seu tempo separado.
+    calc_path_coeff_start = models.DateTimeField(
+        verbose_name="Calc Path Coeff Start",
+        auto_now_add=False,
+        null=True,
+        blank=True,
+        help_text="Start of path coeff."
+    )
+
+    calc_path_coeff_finish = models.DateTimeField(
+        verbose_name="Calc Path Coeff Finish",
+        auto_now_add=False,
+        null=True,
+        blank=True,
+        help_text="End of path coeff."
+    )
+
+    calc_path_coeff_exec_time = models.DurationField(
+        verbose_name="Calc Path Coeff Execution Time",
+        null=True,
+        blank=True,
+        help_text="Execution time of the path coeff step."
+    )
+
     # Etapa de Ingestão de Resultados (prenchimento dessa tabela e da tno_occultation)
 
     # Total de ocultações que foram inseridas no banco de dados

diff --git a/docker-compose-development-template.yml b/docker-compose-development-template.yml
@@ -119,6 +119,29 @@ services:
       - PARSL_ENV=local # PARSL_ENV define onde será o processamento: local ou linea (Slurm)
       - PREDICT_OUTPUTS=/app/outputs/predict_occultation
 
+  predict_occultation:
+    build: ./predict_occultation
+    # Para executar os pipelines
+    # Utilizar daemon.sh Para Executar automaticamente conforme sao criados no portal
+    command: /app/src/daemon.sh
+    # Utilizar sleep infinite Para Executar manualmente
+    # command: /bin/sh -c "while sleep 1000; do :; done"
+    network_mode: "host"
+    volumes:
+      - ./predict_occultation:/app
+      - ./archive:/app/outputs
+    environment:
+      # Acesso ao DB de Catalogo e Administrativo, caso localhost não funcione utilizar o host.docker.internal.
+      # Usando Localhost
+      - DB_URI=postgresql+psycopg2://untrustedprod:untrusted@localhost:3307/prod_gavo
+      - DB_URI_ADMIN=postgresql+psycopg2://tnouseradmindev:TnoDev405060@localhost:3307/tnodbdev
+      # # Usando host.docker.internal
+      # - DB_URI=postgresql+psycopg2://untrustedprod:untrusted@host.docker.internal:3307/prod_gavo
+      # - DB_URI_ADMIN=postgresql+psycopg2://postgres:postgres@host.docker.internal:5432/postgres
+      - USER=apptno
+      - PARSL_ENV=local # PARSL_ENV define onde será o processamento: local ou linea (Slurm)
+      - PREDICT_OUTPUTS=/app/outputs/predict_occultation
+
 volumes:
   rabbitmq-data:
   postgres-data:
diff --git a/docker-compose-production-template.yml b/docker-compose-production-template.yml
@@ -1,9 +1,10 @@
 version: "3.6"
 services:
-  # Backend Django
   backend: &backend
     user: "1000:10000"
-    image: linea/tno:backend_11tech_089d4cb
+    image: linea/tno:backend_d212e5b
+    #image: linea/tno:backend_783abb4
+    #image: linea/tno:backend_11tech_6ae6899
     command: /start.sh
     volumes:
       # Arquivo de Variaveis de ambiente
@@ -31,6 +32,12 @@ services:
     <<: *backend
     command: /start_beat.sh
 
+  celery_flower: &celery_flower
+    <<: *backend
+    command: /start_flower.sh
+    depends_on:
+      - backend
+
   # Esta aberto na rede para que os pipelines possam conectar neste servico.
   rabbit:
     image: rabbitmq:3-management
@@ -55,7 +62,9 @@ services:
 
   # Frontend React/Servidor Web NGINX
   frontend:
-    image: linea/tno:frontend_11tech_089d4cb
+    image: linea/tno:frontend_d212e5b
+    #image: linea/tno:frontend_783abb4
+    #image: linea/tno:frontend_11tech_6ae6899
     ports:
       - 8060:8080
     volumes:
@@ -64,38 +73,35 @@ services:
       - ./data:/var/www/data
     depends_on:
       - backend
-      - jupyter
-      - sora            
+      - rabbit
+      - celery_flower
 
-  sora:
-    image: mozaru/carto-sora
-    user: "1000:10000"
-    environment:
-      INPUT_PATH: "/media/input"
-      OUTPUT_PATH: "/media/output"
-      CACHE_PATH: "/media/output"
-      PORT: 8000
+  pipelines:
+    user: "31670:15010"
+    group_add:
+        - "900"
+    image: linea/tno:pipelines_f2c9b54
+    restart: always
+    command: /app/daemon.sh
+    network_mode: "host"
     volumes:
-      - ./data/sora/input:/media/input
-      - ./data/sora/output:/media/output
-    command: python3.7 /opt/server/main.py
+      - /lustre/t1/scratch/users/app.tno/.ssh/:/home/apptno/.ssh
+      - /lustre/t1/scratch/users/app.tno/tno_testing/outputs/runinfo/:/app/runinfo
+      - /lustre/t1/scratch/users/app.tno/tno_testing/:/lustre/t1/scratch/users/app.tno/tno_testing
+    environment:
+      - DB_URI=postgresql+psycopg2://untrustedprod:untrusted@desdb4.linea.gov.br:5432/prod_gavo
+      - DB_URI_ADMIN=postgresql+psycopg2://tnouseradmindev:TnoDev405060@desdb4.linea.gov.br:5432/tnodbdev
+      - USER=apptno
+      - PARSL_ENV=linea
+      - PREFIX_ENV=testing
+      - PIPELINE_PREDIC_OCC=/lustre/t1/scratch/users/app.tno/tno_testing/predict_occ
+      - PIPELINE_PATH=/lustre/t1/scratch/users/app.tno/tno_testing/predict_occ/pipeline
+      - PREDICT_OUTPUTS=/lustre/t1/scratch/users/app.tno/tno_testing/outputs/predict_occ
+      - SLURM_REMOTE_DIR=/lustre/t1/scratch/users/app.tno/tno_testing/configs
+      - PIPE_BIN=/lustre/t1/scratch/users/app.tno/tno_testing/bin
+      - CONDA_ENVS=/lustre/t1/scratch/users/app.tno/tno_testing/miniconda/envs
+      - SSHKEY=/home/apptno/.ssh/id_rsa
 
-  # #Jupyter Notebook for Science Verification (access database and files)
-  # jupyter:
-  #   image: jupyter/scipy-notebook:latest
-  #   command: "start-notebook.sh"
-  #   user: "1000:10000"
-  #   environment:
-  #     NB_USER: docker_worker
-  #     CHOWN_HOME: 'yes'
-  #     CHOWN_HOME_OPTS: -R
-  #     JUPYTER_ENABLE_LAB: 'yes'
-  #     PASSWORD: "password"
-  #   volumes:
-  #     - ./jupyter_data:/home/jovyan/work
-  #     - ./jupyter_server_config.py:/home/jovyan/.jupyter/jupyter_server_config.py:ro
-  #   ports:
-  #     - 8888:8888      
 volumes:
   rabbitmq-data:
   rabbitmq_portal-data:
diff --git a/frontend/src/pages/PredictOccultation/Asteroid.js b/frontend/src/pages/PredictOccultation/Asteroid.js
@@ -107,6 +107,10 @@ function PredictionAsteroid() {
           title: 'Predict Occultation Execution Time',
           value: predictionJobResult.pre_occ_exec_time?predictionJobResult.pre_occ_exec_time.split('.')[0]:"-"
         },
+        {
+          title: 'Path Coeff Execution Time',
+          value: predictionJobResult.calc_path_coeff_exec_time?predictionJobResult.calc_path_coeff_exec_time.split('.')[0]:"-"
+        },        
         {
           title: 'Result Ingestion Execution Time',
           value: predictionJobResult.ing_occ_exec_time?predictionJobResult.ing_occ_exec_time.split('.')[0]:"-"

diff --git a/orbit_trace/.dockerignore b/orbit_trace/.dockerignore
@@ -0,0 +1,6 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+*.pyc
+*.pyo