From 56395b65b83f00b91e8efe71bd876fd70b36b7d0 Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Mon, 3 Jun 2024 12:02:03 -0400 Subject: [PATCH 1/5] unpin awscli --- .github/workflows/update-testing-data.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-testing-data.yml b/.github/workflows/update-testing-data.yml index f047ce4b..cbf342cc 100644 --- a/.github/workflows/update-testing-data.yml +++ b/.github/workflows/update-testing-data.yml @@ -38,7 +38,7 @@ jobs: - if: steps.cache-ophys-datasets.outputs.cache-hit == false name: Install and configure AWS CLI run: | - pip install awscli==1.29.56 + pip install awscli aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} - if: steps.cache-ophys-datasets.outputs.cache-hit == false From 794b6a72dd97f468e8b08b8c10d570852bca7a49 Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Mon, 3 Jun 2024 12:07:11 -0400 Subject: [PATCH 2/5] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2287c6ae..ad9780b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ * Remove unnecessary `scipy` import error handling: [#315](https://github.com/catalystneuro/roiextractors/pull/315) * Fixed the typing returned by the `InscopixImagingExtractor.get_dtype` method: [#326](https://github.com/catalystneuro/roiextractors/pull/326) * Detect Changelog Updates was moved to its own dedicated workflow to avoid daily testing failures: [#336](https://github.com/catalystneuro/roiextractors/pull/336) +* Fixed the Update Testing Data Workflow by unpinning the version of awscli: [#339](https://github.com/catalystneuro/roiextractors/pull/339) ### Improvements From bf2acea47ebe447616c09762b85402ea75b6cb5d Mon Sep 17 00:00:00 2001 From: Paul Adkisson Date: Mon, 3 Jun 2024 12:19:43 -0400 Subject: [PATCH 3/5] Added conditional download block to tests workflow --- .github/workflows/run-tests.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 9aa460cb..b7547a33 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -51,6 +51,16 @@ jobs: with: path: ./ophys_testing_data key: ophys-datasets-042023-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} + + - if: steps.cache-ophys-datasets.outputs.cache-hit == false + name: Install and configure AWS CLI + run: | + pip install awscli + aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} + aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - if: steps.cache-ophys-datasets.outputs.cache-hit == false + name: Download data from S3 + run: aws s3 cp --recursive s3://${{ secrets.S3_GIN_BUCKET }}//ophys_testing_data ./ophys_testing_data - name: Run full pytest with coverage run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml From 30f97cff692d7eb30160eaa2ea6bde4338a47752 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 3 Jun 2024 16:19:50 +0000 Subject: [PATCH 4/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index b7547a33..a43d5442 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -51,7 +51,7 @@ jobs: with: path: ./ophys_testing_data key: ophys-datasets-042023-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - + - if: steps.cache-ophys-datasets.outputs.cache-hit == false name: Install and configure AWS CLI run: | From 8431a5d97b4e9ae5d099b8a729d2b4bcfef41a38 Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 3 Jun 2024 09:38:04 -0700 Subject: [PATCH 5/5] refactored pr tests to a dedicated workflow to streamline data download --- .github/workflows/deploy_pr_tests.yaml | 21 +++++++++++++++++++++ .github/workflows/dev-testing.yaml | 6 +----- .github/workflows/run-tests.yml | 15 --------------- 3 files changed, 22 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/deploy_pr_tests.yaml diff --git a/.github/workflows/deploy_pr_tests.yaml b/.github/workflows/deploy_pr_tests.yaml new file mode 100644 index 00000000..58c040c1 --- /dev/null +++ b/.github/workflows/deploy_pr_tests.yaml @@ -0,0 +1,21 @@ +name: Deploy PR Tests + +on: + workflow_dispatch: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + update-testing-data: + uses: ./.github/workflows/update-testing-data.yml + + run-tests: + needs: update-testing-data + uses: ./.github/workflows/run-tests.yml + + dev-tests: + needs: update-testing-data + uses: ./.github/workflows/dev-testing.yaml diff --git a/.github/workflows/dev-testing.yaml b/.github/workflows/dev-testing.yaml index 65d595ff..af695647 100644 --- a/.github/workflows/dev-testing.yaml +++ b/.github/workflows/dev-testing.yaml @@ -1,11 +1,7 @@ name: Dev Testing on: - pull_request: workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + workflow_call: jobs: run: diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index a43d5442..ab173ca5 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -1,13 +1,8 @@ name: Full Tests on: - pull_request: workflow_dispatch: workflow_call: -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - jobs: run: name: Full tests on ${{ matrix.os }} with Python ${{ matrix.python-version }} @@ -52,16 +47,6 @@ jobs: path: ./ophys_testing_data key: ophys-datasets-042023-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} - - if: steps.cache-ophys-datasets.outputs.cache-hit == false - name: Install and configure AWS CLI - run: | - pip install awscli - aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} - aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - if: steps.cache-ophys-datasets.outputs.cache-hit == false - name: Download data from S3 - run: aws s3 cp --recursive s3://${{ secrets.S3_GIN_BUCKET }}//ophys_testing_data ./ophys_testing_data - - name: Run full pytest with coverage run: pytest -n auto --dist loadscope --cov=./ --cov-report xml:./codecov.xml - if: ${{ matrix.python-version == '3.9' && matrix.os == 'ubuntu-latest'}}