upload smoke test files (#4)

* upload Calcos smoke test * test by python version * fix CRDS path * fix filename * replace requirement within environment file * rename key * rename jobs * replace Python version * replace Python version * use cache in smoke test * cache packages separately * cache Python version environments separately * fix cache paths * unpin Python patch version * unpin Python patch version * refactor date
spacetelescope · Jul 15, 2022 · 5408f90 · 5408f90
1 parent cc5b08b
commit 5408f90
Show file tree

Hide file tree

Showing 5 changed files with 78 additions and 129 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,2 @@
+*.fits filter=lfs diff=lfs merge=lfs -text
+*.asdf filter=lfs diff=lfs merge=lfs -text
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -19,40 +19,48 @@ env:
 
 jobs:
   build:
-    name: build environment
+    name: build environment (Python ${{ matrix.python }})
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python: [ '3.9', '3.10' ]
     steps:
       - uses: actions/checkout@main
+      - run: sed -i "s/python==3.9/python==${{ matrix.python }}.*/g" spacetelescope-env-latest.yml
       - uses: mtkennerly/dunamai-action@master
         id: version
-      - run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+      - run: echo "::set-output name=date::$(date +'%Y%m%d')"
         id: date
       - uses: actions/cache@main
         with:
-          path: |
-            ~/conda_pkgs_dir
-            /usr/share/miniconda/envs/spacetelescope-env-latest
-          key: conda-${{ runner.os }}-${{ steps.date.outputs.date }}
+          path: /home/runner/conda_pkgs_dir
+          key: conda-packages-${{ steps.date.outputs.date }}
+      - uses: actions/cache@main
+        with:
+          path: /usr/share/miniconda/envs/spacetelescope-env-${{ matrix.python }}-latest
+          key: conda-${{ runner.os }}-${{ matrix.python }}-${{ steps.date.outputs.date }}
       - uses: conda-incubator/setup-miniconda@main
         with:
-          activate-environment: spacetelescope-env-latest
+          activate-environment: spacetelescope-env-${{ matrix.python }}-latest
           environment-file: spacetelescope-env-latest.yml
+          python-version: ${{ matrix.python }}.*
+          auto-update-conda: true
           use-only-tar-bz2: true
       - run: conda update --all
-      - run: conda env export > spacetelescope-env-${{ steps.version.outputs.version }}.yml
-      - run: sed -i "s/spacetelescope-env-latest/spacetelescope-env-${{ steps.version.outputs.version }}/g" spacetelescope-env-${{ steps.version.outputs.version }}.yml
-      - run: cat spacetelescope-env-${{ steps.version.outputs.version }}.yml
+      - run: conda env export > spacetelescope-env-${{ matrix.python }}-${{ steps.version.outputs.version }}.yml
+      - run: sed -i "s/spacetelescope-env-${{ matrix.python }}-latest/spacetelescope-env-${{ matrix.python }}-${{ steps.version.outputs.version }}/g" spacetelescope-env-${{ matrix.python }}-${{ steps.version.outputs.version }}.yml
+      - run: cat spacetelescope-env-${{ matrix.python }}-${{ steps.version.outputs.version }}.yml
       - uses: actions/upload-artifact@main
         with:
-          name: spacetelescope-env-${{ steps.version.outputs.version }}.yml
-          path: spacetelescope-env-${{ steps.version.outputs.version }}.yml
+          name: spacetelescope-env-${{ matrix.python }}-${{ steps.version.outputs.version }}.yml
+          path: spacetelescope-env-${{ matrix.python }}-${{ steps.version.outputs.version }}.yml
   run_packaged_unit_tests:
-    name: run `${{ matrix.package }}` unit tests
+    name: run `${{ matrix.package }}` unit tests (Python ${{ matrix.python }})
     needs: [ build ]
     runs-on: ubuntu-latest
     strategy:
-      fail-fast: false
       matrix:
+        python: [ '3.9', '3.10' ]
         include:
           - package: acstools
             dependencies: [ ci-watson, pytest-astropy-header, pytest-remotedata ]
@@ -78,28 +86,33 @@ jobs:
             dependencies: [ ]
     steps:
       - uses: actions/checkout@main
-      - run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+      - run: sed -i "s/python==3.9/python==${{ matrix.python }}.*/g" spacetelescope-env-latest.yml
+      - run: echo "::set-output name=date::$(date +'%Y%m%d')"
         id: date
       - uses: actions/cache@main
         with:
-          path: |
-            ~/conda_pkgs_dir
-            /usr/share/miniconda/envs/spacetelescope-env-latest
-          key: conda-${{ runner.os }}-${{ steps.date.outputs.date }}
+          path: /home/runner/conda_pkgs_dir
+          key: conda-packages-${{ steps.date.outputs.date }}
+      - uses: actions/cache@main
+        with:
+          path: /usr/share/miniconda/envs/spacetelescope-env-${{ matrix.python }}-latest
+          key: conda-${{ runner.os }}-${{ matrix.python }}-${{ steps.date.outputs.date }}
       - uses: conda-incubator/setup-miniconda@main
         with:
-          activate-environment: spacetelescope-env-latest
+          activate-environment: spacetelescope-env-${{ matrix.python }}-latest
           environment-file: spacetelescope-env-latest.yml
+          python-version: ${{ matrix.python }}.*
+          auto-update-conda: true
           use-only-tar-bz2: true
       - run: pip install pytest-xdist ${{ join(matrix.dependencies, ' ') }}
       - run: pytest -n auto --pyargs ${{ matrix.package }}
   run_repository_unit_tests:
-    name: run `${{ matrix.package }}` unit tests
+    name: run `${{ matrix.package }}` unit tests (Python ${{ matrix.python }})
     needs: [ build ]
     runs-on: ubuntu-latest
     strategy:
-      fail-fast: false
       matrix:
+        python: [ '3.9', '3.10' ]
         include:
           - package: calcos
             repository: spacetelescope/calcos
@@ -121,105 +134,82 @@ jobs:
       - uses: actions/checkout@main
         with:
           path: spacetelescope-env-distribution
-      - run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
+      - run: sed -i "s/python==3.9/python==${{ matrix.python }}.*/g" spacetelescope-env-distribution/spacetelescope-env-latest.yml
+      - run: echo "::set-output name=date::$(date +'%Y%m%d')"
         id: date
       - uses: actions/cache@main
         with:
-          path: |
-            ~/conda_pkgs_dir
-            /usr/share/miniconda/envs/spacetelescope-env-latest
-          key: conda-${{ runner.os }}-${{ steps.date.outputs.date }}
+          path: /home/runner/conda_pkgs_dir
+          key: conda-packages-${{ steps.date.outputs.date }}
+      - uses: actions/cache@main
+        with:
+          path: /usr/share/miniconda/envs/spacetelescope-env-${{ matrix.python }}-latest
+          key: conda-${{ runner.os }}-${{ matrix.python }}-${{ steps.date.outputs.date }}
       - uses: conda-incubator/setup-miniconda@main
         with:
-          activate-environment: spacetelescope-env-latest
+          activate-environment: spacetelescope-env-${{ matrix.python }}-latest
           environment-file: spacetelescope-env-distribution/spacetelescope-env-latest.yml
+          python-version: ${{ matrix.python }}.*
+          auto-update-conda: true
           use-only-tar-bz2: true
       - uses: actions/checkout@main
         with:
           path: ${{ matrix.package }}
           repository: ${{ matrix.repository }}
       - run: cd ${{ matrix.package }} && pip install pytest-xdist ${{ join(matrix.dependencies, ' ') }}
       - run: cd ${{ matrix.package }} && pytest -n auto ${{ matrix.test_directory }}
-  download_data:
-    name: download data for smoke tests
-    needs: [ run_packaged_unit_tests, run_repository_unit_tests ]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@main
-      - uses: actions/setup-python@main
-        with:
-          python-version: '3.x'
-      - uses: actions/cache@main
-        with:
-          path: 'tests/data/*.fits'
-          key: data-${{ hashFiles('tests/data/download.py') }}
-      - run: pip install pooch
-      - run: python tests/data/download.py
   run_calcos_smoke_tests:
-    name: run `calcos` smoke tests
-    needs: [ download_data ]
+    name: run `calcos` smoke tests (Python ${{ matrix.python }})
+    needs: [ build ]
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python: [ '3.9', '3.10' ]
     steps:
       - uses: actions/checkout@main
         with:
           lfs: true
+      - run: sed -i "s/python==3.9/python==${{ matrix.python }}.*/g" spacetelescope-env-latest.yml
+      - run: echo "::set-output name=date::$(date +'%Y%m%d')"
+        id: date
       - uses: actions/cache@main
         with:
-          path: |
-            ~/conda_pkgs_dir
-            /usr/share/miniconda/envs/spacetelescope-env-latest
-          key: conda-${{ runner.os }}-${{ github.sha }}
-      - uses: conda-incubator/setup-miniconda@main
-        with:
-          activate-environment: spacetelescope-env-latest
-          environment-file: spacetelescope-env-latest.yml
-          use-only-tar-bz2: true
+          path: /home/runner/conda_pkgs_dir
+          key: conda-packages-${{ steps.date.outputs.date }}
       - uses: actions/cache@main
         with:
-          path: 'tests/data/*.fits'
-          key: conda-${{ hashFiles('tests/data/download.py') }}
-      - run: calcos la8n01qkq_rawtag_a.fits
-        env:
-          lref: /grp/crds/hst/references/hst/
-  run_drizzlepac_smoke_tests:
-    name: run `drizzlepac` smoke tests
-    needs: [ download_data ]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@main
-        with:
-          lfs: true
-      - uses: actions/cache@main
-        with:
-          path: |
-            ~/conda_pkgs_dir
-            /usr/share/miniconda/envs/spacetelescope-env-latest
-          key: conda-${{ runner.os }}-${{ github.sha }}
+          path: /usr/share/miniconda/envs/spacetelescope-env-${{ matrix.python }}-latest
+          key: conda-${{ runner.os }}-${{ matrix.python }}-${{ steps.date.outputs.date }}
       - uses: conda-incubator/setup-miniconda@main
         with:
-          activate-environment: spacetelescope-env-latest
+          activate-environment: spacetelescope-env-${{ matrix.python }}-latest
           environment-file: spacetelescope-env-latest.yml
+          python-version: ${{ matrix.python }}.*
+          auto-update-conda: true
           use-only-tar-bz2: true
       - uses: actions/cache@main
         with:
-          path: 'tests/data/*.fits'
-          key: conda-${{ hashFiles('tests/data/download.py') }}
-      - run: python tests/test_drizzlepac.py
+          path: 'tests/data/'
+          key: conda-${{ hashFiles('tests/data/*') }}
+      - run: calcos la8n01qkq_rawtag_a.fits
         env:
-          jref: /grp/crds/hst/references/hst/
+          jref: /home/runner/crds_cache/hst/references/hst/
   export:
     name: export environment
-    needs: [ run_packaged_unit_tests, run_repository_unit_tests ]
+    needs: [ run_packaged_unit_tests, run_repository_unit_tests, run_calcos_smoke_tests ]
     if: github.event_name == 'release' && github.event.action == 'published'
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python: [ '3.9', '3.10' ]
     steps:
       - uses: actions/checkout@main
       - uses: mtkennerly/dunamai-action@master
         id: version
       - uses: dawidd6/action-download-artifact@main
         with:
-          name: spacetelescope-env-${{ steps.version.outputs.version }}.yml
-          path: spacetelescope-env-${{ steps.version.outputs.version }}.yml
-      - uses: svenstaro/upload-release-action@2.3.0
+          name: spacetelescope-env-${{ matrix.python }}-${{ steps.version.outputs.version }}.yml
+          path: spacetelescope-env-${{ matrix.python }}-${{ steps.version.outputs.version }}.yml
+      - uses: svenstaro/upload-release-action@master
         with:
-          file: spacetelescope-env-${{ steps.version.outputs.version }}.yml
+          file: spacetelescope-env-${{ matrix.python }}-${{ steps.version.outputs.version }}.yml
diff --git a/tests/data/download.py b/tests/data/download.py
diff --git a/tests/data/la8n01qkq_rawtag_a.fits b/tests/data/la8n01qkq_rawtag_a.fits
diff --git a/tests/test_drizzlepac.py b/tests/test_drizzlepac.py
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		*.fits filter=lfs diff=lfs merge=lfs -text
		*.asdf filter=lfs diff=lfs merge=lfs -text