Merge pull request #136 from templateflow/maint/circleci

MAINT: Overdue updates and revision of CircleCI
templateflow · Sep 13, 2024 · 84a672b · 84a672b
2 parents c96b2cb + 58054df
commit 84a672b
Show file tree

Hide file tree

Showing 17 changed files with 431 additions and 442 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,44 +1,85 @@
-version: 2
+version: 2.1
+orbs:
+  docker: circleci/docker@2.7.0
+  codecov: codecov/codecov@4.1.0
+  python: circleci/python@2.1.1
+
 jobs:
-  tests:
-    docker:
-      - image: cimg/python:3.11
-        auth:
-          username: $DOCKER_USER
-          password: $DOCKER_PAT
-    working_directory: /tmp/tests
-    environment:
-      - OSF_MIRROR_PATH: /tmp/data/templateflow
-    steps:
-      - checkout:
-          path: /tmp/src/templateflow
+  build:
+    executor:
+      name: python/default
+      tag: '3.12'
+
+    # docker:
+    #   - auth:
+    #       username: $DOCKER_USERNAME
+    #       password: $DOCKER_PASSWORD
 
+    steps:
+      - checkout
       - run:
-          name: Generate requirements.txt
           command: |
-            python /tmp/src/templateflow/.maint/update_requirements.py
-
-      - restore_cache:
-          keys:
-            - deps-v11-{{ checksum "/tmp/src/templateflow/dev-requirements.txt"}}-{{ epoch }}
-            - deps-v11-{{ checksum "/tmp/src/templateflow/dev-requirements.txt"}}-
-            - deps-v11-
+            echo 'build' > requirements.txt
+            echo 'wheel' >> requirements.txt
+            echo 'twine' >> requirements.txt
+      - python/install-packages:
+          pkg-manager: pip
+          cache-version: build-v1
       - run:
-          name: Prepare environment
           command: |
-            python -m venv /tmp/venv
-            source /tmp/venv/bin/activate
-            python -m pip install -U pip
-            python -m pip install -r /tmp/src/templateflow/dev-requirements.txt
-            python -m pip install -U "datalad ~= 0.19.0"
-            python -m pip install -U build hatch twine pkginfo codecov
+            python -m build
+            python -m twine check dist/*
+          name: Test packaging
+      - persist_to_workspace:
+          root: ~/project
+          paths:
+            - .
+
+  deploy_pypi:
+    executor:
+      name: python/default
+      tag: '3.12'
+
+    # docker:
+    #   - auth:
+    #       username: $DOCKER_USERNAME
+    #       password: $DOCKER_PASSWORD
+
+    steps:
+      - attach_workspace:
+          at: ~/project
+
+      - python/install-packages:
+          args: twine
+          pkg-manager: pip
 
+      - run:
+          command: python -m twine upload dist/*
+          name: Upload to Pypi
+
+  tests:
+    executor:
+      name: python/default
+      tag: '3.12'
+
+    # docker:
+    #   - auth:
+    #       username: $DOCKER_USERNAME
+    #       password: $DOCKER_PASSWORD
+
+    environment:
+      - OSF_MIRROR_PATH: /tmp/data/templateflow
+    steps:
+      - restore_cache:
+          keys:
+            - annex-v1-{{ epoch }}
+            - annex-v1-
       - run:
           name: Install git and git-annex
           command: |
             if [[ ! -e "/tmp/cache/git-annex-standalone.tar.gz" ]]; then
               wget -O- http://neuro.debian.net/lists/focal.us-ca.full | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list
-              sudo apt-key add /tmp/src/templateflow/.neurodebian/neurodebian.gpg
+              sudo apt-key add .neurodebian/neurodebian.gpg
               sudo apt-key adv --recv-keys --keyserver hkps://keys.openpgp.org 0xA5D32F012649A5A9 || true
               sudo apt update && sudo apt-get install -y --no-install-recommends git-annex-standalone
               mkdir -p /tmp/cache
@@ -50,83 +91,80 @@ jobs:
             git config --global user.email "email@domain.com"
 
       - save_cache:
-          key: deps-v11-{{ checksum "/tmp/src/templateflow/dev-requirements.txt"}}-{{ epoch }}
+          key: annex-v1-{{ epoch }}
           paths:
             - "/tmp/cache"
-            - "/tmp/venv"
+
+      - attach_workspace:
+          at: ~/project
 
       - run:
-          name: Test packaging
           command: |
-            source /tmp/venv/bin/activate
-            cd /tmp/src/templateflow
-            python -m build
-            python -m twine check dist/*
+            python .maint/update_requirements.py
+          name: Generate requirements.txt
+
+      - python/install-packages:
+          pip-dependency-file: dev-requirements.txt
+          pkg-manager: pip
+          cache-version: v1
 
       - run:
-          name: Run tests (w/ DataLad)
+          name: Run tests (config and others)
           command: |
-            source /tmp/venv/bin/activate
-            export TEMPLATEFLOW_USE_DATALAD=on
-            pytest --junit-xml=/tmp/tests/datalad.xml --cov templateflow --cov-report xml:/tmp/cov_api_dl.xml \
-                   --doctest-modules /tmp/src/templateflow/templateflow
+            python -m pytest \
+                   --junit-xml=~/tests/conftests.xml --cov templateflow --cov-report xml:~/coverage/cov_config.xml \
+                   templateflow/tests/
+
+      - codecov/upload:
+          file: ~/coverage/cov_config.xml
+          flags: config
+          upload_name: General
 
       - run:
-          name: Submit api test coverage
+          name: Run tests (w/ DataLad)
           command: |
-            source /tmp/venv/bin/activate
-            codecov --file /tmp/cov_api_dl.xml --root /tmp/src/templateflow \
-                --flags api,datalad -e CIRCLE_JOB
+            mkdir -p ~/tests/ ~/coverage/
+            export TEMPLATEFLOW_USE_DATALAD=on
+            python -m pytest \
+                   --junit-xml=~/tests/datalad.xml --cov templateflow --cov-report xml:~/coverage/cov_api_dl.xml \
+                   --doctest-modules templateflow/api.py
+
+      - codecov/upload:
+          file: ~/coverage/cov_api_dl.xml
+          flags: api,datalad
+          upload_name: Datalad tests
 
       - run:
           name: Run tests (pulling from S3)
           command: |
-            source /tmp/venv/bin/activate
             export TEMPLATEFLOW_USE_DATALAD=off
             export TEMPLATEFLOW_HOME=$HOME/templateflow-s3
-            pytest --junit-xml=/tmp/tests/s3.xml --cov templateflow --cov-report xml:/tmp/cov_api_s3.xml \
-                   --doctest-modules /tmp/src/templateflow/templateflow/api.py
+            python -m pytest \
+                   --junit-xml=~/tests/s3.xml --cov templateflow --cov-report xml:~/coverage/cov_api_s3.xml \
+                   --doctest-modules templateflow/api.py
 
-      - run:
-          name: Submit api test coverage
-          command: |
-            source /tmp/venv/bin/activate
-            codecov --file /tmp/cov_api_s3.xml --root /tmp/src/templateflow \
-                --flags api,s3 -e CIRCLE_JOB
+      - codecov/upload:
+          file: ~/coverage/cov_api_s3.xml
+          flags: api,s3
+          upload_name: S3 tests
 
       - run:
           name: Run tests (w/ DataLad, bypassed via S3)
           command: |
-            source /tmp/venv/bin/activate
             export TEMPLATEFLOW_USE_DATALAD=off
             export TEMPLATEFLOW_HOME=$HOME/templateflow-clean
             datalad install -r -s https://github.com/templateflow/templateflow $TEMPLATEFLOW_HOME
-            pytest --junit-xml=/tmp/tests/dl+s3.xml --cov templateflow --cov-report xml:/tmp/cov_api_dl_s3.xml \
-                   --doctest-modules /tmp/src/templateflow/templateflow/api.py
+            python -m pytest \
+                   --junit-xml=~/tests/dl+s3.xml --cov templateflow --cov-report xml:~/coverage/cov_api_dl_s3.xml \
+                   --doctest-modules templateflow/api.py
 
-      - run:
-          name: Submit api test coverage
-          command: |
-            source /tmp/venv/bin/activate
-            codecov --file /tmp/cov_api_dl_s3.xml --root /tmp/src/templateflow \
-                --flags api,dls3 -e CIRCLE_JOB
-
-      - run:
-          name: Run tests (config, parameterized TEMPLATEFLOW_USE_DATALAD)
-          command: |
-            source /tmp/venv/bin/activate
-            pytest --junit-xml=/tmp/tests/conftests.xml --cov templateflow --cov-report xml:/tmp/cov_config.xml \
-                   /tmp/src/templateflow/templateflow/conf/tests/test_conf.py
-
-      - run:
-          name: Submit config test coverage
-          command: |
-            source /tmp/venv/bin/activate
-            codecov --file /tmp/cov_config.xml --root /tmp/src/templateflow \
-                --flags config -e CIRCLE_JOB
+      - codecov/upload:
+          file: ~/coverage/cov_api_dl_s3.xml
+          flags: api,dls3
+          upload_name: Datalad-S3-bypass
 
       - store_test_results:
-          path: /tmp/tests
+          path: ~/tests
 
   build_docs:
     machine:
@@ -179,54 +217,38 @@ jobs:
       - store_artifacts:
           path: ~/html
 
-  deploy_pypi:
-    docker:
-      - image: cimg/python:3.9
-    working_directory: /tmp/src/templateflow
-    steps:
-      - attach_workspace:
-          at: /tmp
-
-      - checkout:
-          path: /tmp/src/templateflow
-
-      - run:
-          name: Generate requirements.txt
-          command: |
-            python /tmp/src/templateflow/.maint/update_requirements.py
-
-      - restore_cache:
-          keys:
-            - deps-v11-{{ checksum "/tmp/src/templateflow/dev-requirements.txt"}}-{{ epoch }}
-            - deps-v11-{{ checksum "/tmp/src/templateflow/dev-requirements.txt"}}-
-            - deps-v11-
-
-      - run:
-          name: Deploy to PyPi
-          command: |
-            source /tmp/venv/bin/activate
-            python -m pip install build -U twine pkginfo
-            python -m build
-            python -m twine check dist/*
-            python -m twine upload dist/*
-
 workflows:
   version: 2
   build_test_deploy:
     jobs:
+      - build:
+          context:
+            - nipreps-common
+          filters:
+            branches:
+              ignore:
+                - /docs?\/.*/
+            tags:
+              only: /.*/
+
       - tests:
           context:
             - nipreps-common
+          requires:
+            - build
           filters:
             branches:
               ignore:
                 - /docs?\/.*/
             tags:
               only: /.*/
+
       - deploy_pypi:
           context:
             - nipreps-common
           requires:
+            - build
+            - tests
             - build_docs
           filters:
             branches:

diff --git a/.maint/update_requirements.py b/.maint/update_requirements.py
@@ -16,13 +16,13 @@
 reqs_dev = repo_root / 'dev-requirements.txt'
 
 requirements = [
-    Requirement(req)
-    for req in loads(pyproject.read_text())['project']['dependencies']
+    Requirement(req) for req in loads(pyproject.read_text())['project']['dependencies']
 ]
 
 requirements_dev = [
     Requirement(req)
-    for req in loads(pyproject.read_text())['project']['optional-dependencies']['test']
+    for group in ('test', 'datalad')
+    for req in loads(pyproject.read_text())['project']['optional-dependencies'][group]
 ]
 
 script_name = Path(__file__).relative_to(repo_root)

diff --git a/pyproject.toml b/pyproject.toml
@@ -37,13 +37,15 @@ Documentation = "https://www.templateflow.org/python-client/"
 
 [project.optional-dependencies]
 test = [
+    "coverage ~= 5.0.0",
     "pytest",
+    "pytest-cov",
+    "pytest-env",
     "pytest-xdist",
-    "pytest-cov == 2.5.1",
-    "coverage",
+    "toml",
 ]
 datalad = [
-    "datalad ~= 0.19.0"
+    "datalad ~= 1.0.0"
 ]
 doc = [
     "nbsphinx",
@@ -113,15 +115,15 @@ per-file-ignores = [
 
 [tool.pytest.ini_options]
 norecursedirs = [".git"]
-addopts = "-svx --doctest-modules"
+addopts = "-svx"
 doctest_optionflags = "ALLOW_UNICODE NORMALIZE_WHITESPACE ELLIPSIS"
 env = "PYTHONHASHSEED=0"
 filterwarnings = ["ignore::DeprecationWarning"]
 junit_family = "xunit2"
 
 [tool.coverage.run]
 branch = true
-concurrency = 'multiprocessing'
+concurrency = ['multiprocessing']
 omit = [
     '*/tests/*',
     '*/conftest.py',

diff --git a/templateflow/__init__.py b/templateflow/__init__.py
@@ -21,6 +21,7 @@
 #     https://www.nipreps.org/community/licensing/
 #
 """TemplateFlow is the Zone of Templates."""
+
 from datetime import datetime as _dt
 from datetime import timezone as _tz
 
@@ -30,6 +31,7 @@
     from ._version import __version__
 except ModuleNotFoundError:
     from importlib.metadata import PackageNotFoundError, version
+
     try:
         __version__ = version(__packagename__)
     except PackageNotFoundError:

diff --git a/templateflow/_loader.py b/templateflow/_loader.py
@@ -24,6 +24,7 @@
 
 .. autoclass:: Loader
 """
+
 from __future__ import annotations
 
 import atexit