Fixes for reference data pipeline

azure-pipelines/compare-refdata.yml

@@ -1,41 +1,43 @@
-# Starter pipeline
-# Start with a minimal pipeline that you can customize to build and deploy your code.
-# Add steps that build, run tests, deploy, and more:
-# https://aka.ms/yaml
+# Keep this line, is useful for testing without changing the notebook.
+# sed -i "s/ref2_hash='upstream\/pr\/24'/ref1_hash='c998f44', ref2_hash='master'/g" ref_data_compare.ipynb
 
+# This pipeline runs manually
 trigger: none
-pr: none
+pr: [ master ]
 
 variables:
-  system.debug: "true"
-  ref.data.home: "$(Agent.BuildDirectory)/tardis-refdata"
-  ref.data.github.url: "https://github.com/tardis-sn/tardis-refdata.git"
-  tardis.build.dir: $(Build.Repository.LocalPath)
+  package.manager: "mamba"
+  build.dir: "$(Agent.BuildDirectory)"
+  ref.data.url: "https://github.com/tardis-sn/tardis-refdata.git"
+  ref.data.dir: "$(build.dir)/tardis-refdata"
+  system.debug: "false"
 
 jobs:
   - job: report
     pool:
       vmImage: "ubuntu-latest"
 
     steps:
-      - bash: |
-          echo "##vso[task.prependpath]$CONDA/bin"
-          sudo chown -R $USER $CONDA
-        displayName: "Add CONDA to path"
+      - bash: echo "##vso[task.setvariable variable=shellopts]errexit"
+        displayName: "Force exit on error (bash)"
 
-      - bash: |
-          sh ci-helpers/install_tardis_env.sh
-        displayName: "Install TARDIS env"
+      - bash: echo "##vso[task.prependpath]$CONDA/bin"
+        displayName: "Add conda to path"
 
-      - bash: |
-          source activate tardis
-          conda install bokeh -c conda-forge --no-update-deps --yes
-        displayName: "Install Bokeh"
+      - bash: conda install mamba -c conda-forge -y
+        displayName: "Install Mamba package manager"
+        condition: eq(variables['package.manager'], 'mamba')
+
+      - bash: $PACKAGE_MANAGER env create -f tardis_env3.yml
+        displayName: "Install TARDIS environment"
 
       - bash: |
           source activate tardis
-          git clone https://github.com/tardis-sn/tardis-refdata.git $(ref.data.home)
-        displayName: "Fetch ALL reference data"
+          $PACKAGE_MANAGER install bokeh=1.4 -c conda-forge --no-update-deps -y
+        displayName: "Install Bokeh in TARDIS environment"
+
+      - bash: sh ci-helpers/fetch_reference_data.sh
+        displayName: "Fetch reference data"
 
       - bash: |
           source activate tardis
@@ -44,30 +46,38 @@ jobs:
 
       - bash: |
           source activate tardis
-          python setup.py test --args="--tardis-refdata=$(ref.data.home) --generate-reference"
+          pytest tardis --tardis-refdata=$REF_DATA_DIR --generate-reference
         displayName: "Generate new reference data"
 
       - bash: |
           source activate tardis
-          cd $(ref.data.home)
-          git remote add upstream https://github.com/tardis-sn/tardis-refdata.git
-          git fetch upstream
+          cd $REF_DATA_DIR
           git fetch upstream "+refs/pull/*/head:refs/remotes/upstream/pr/*"
-          cd notebooks
+          git checkout upstream/pr/28
+          git lfs pull --include="unit_test_data.h5" upstream
+        displayName: "Checkout"
+
+      - bash: |
+          source activate tardis
+          cd $REF_DATA_DIR/notebooks
+          jupyter nbconvert ref_data_compare.ipynb --to html --execute --ExecutePreprocessor.timeout=6000
+        displayName: "Generating HTML report"
+
+      - bash: |
+          source activate tardis
+          cd $REF_DATA_DIR/notebooks
           jupyter nbconvert ref_data_compare.ipynb --to html --execute --allow-errors --ExecutePreprocessor.timeout=6000
-        displayName: "Generating report"
+        displayName: "Generating (failed) HTML report"
+        condition: failed()
 
       - task: PublishPipelineArtifact@1
         inputs:
-          targetPath: "$(ref.data.home)/notebooks/ref_data_compare.html"
-          artifact: "report"
+          targetPath: "$(ref.data.dir)/notebooks/ref_data_compare.html"
+          artifact: "html_report"
           publishLocation: "pipeline"
-
-      - bash: |
-          ls -lR /tmp
-          echo
+        displayName: "Publishing artifact"
         condition: succeededOrFailed()
-        displayName: "Check files in /tmp"
 
-# Keep this line, is useful for testing without changing the notebook.
-# sed -i "s/ref2_hash='upstream\/pr\/24'/ref1_hash='c998f44', ref2_hash='master'/g" ref_data_compare.ipynb
+      - bash: ls -lR /tmp
+        displayName: "Check files in /tmp"
+        condition: succeededOrFailed()

ci-helpers/fetch_reference_data.sh

@@ -1,22 +1,30 @@
 #!/usr/bin/env bash
 git lfs install --skip-smudge
 
-if test -e $REF_DATA_HOME; then
-    echo "Ref data available"
+if test -e $REF_DATA_DIR; then
+    echo "Reference data available"
+
 else
-    echo git clone $REF_DATA_GITHUB_URL $REF_DATA_HOME
-    git clone $REF_DATA_GITHUB_URL $REF_DATA_HOME
-
-cd $REF_DATA_HOME
-# Use the following to get the ref-data from the master;
-git fetch origin
-git checkout origin/master
-# Use the following to get the ref-data from a specific pull request
-#git fetch origin pull/20/head:update-ref
-#git checkout update-ref
-git lfs pull --include="atom_data/kurucz_cd23_chianti_H_He.h5" origin
-git lfs pull --include="atom_data/chianti_He.h5" origin
-git lfs pull --include="plasma_reference/" origin
-git lfs pull --include="unit_test_data.h5" origin
-echo MD5 `md5sum unit_test_data.h5`; fi
-cd $TARDIS_BUILD_DIR
+    git clone $REF_DATA_URL $REF_DATA_DIR -o upstream
+    cd $REF_DATA_DIR
+
+    # Use the following to get the ref-data from the master;
+    git fetch upstream
+    git checkout upstream/master
+
+    # Use the following to get the ref-data from a specific pull request
+    # git fetch upstream pull/20/head:update-ref
+    # git checkout update-ref
+
+    git lfs pull --include="atom_data/kurucz_cd23_chianti_H_He.h5" upstream
+    git lfs pull --include="atom_data/chianti_He.h5" upstream
+    git lfs pull --include="plasma_reference/" upstream
+    git lfs pull --include="unit_test_data.h5" upstream
+    git lfs pull --include="packet_unittest.h5" upstream
+
+    echo MD5: `md5sum unit_test_data.h5`
+    echo MD5: `md5sum atom_data/kurucz_cd23_chianti_H_He.h5`
+
+fi
+
+exit 0