[WIP] Add duration scripts and logic from conda repo

.github/workflows/durations.yml

@@ -0,0 +1,63 @@
+name: Update Durations
+
+on:
+  # every Sunday at 00:00 UTC
+  # https://crontab.guru/#0_0_*_*_0
+  schedule:
+    - cron: '0 0 * * 0'
+
+  # https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_dispatch
+  workflow_dispatch:
+
+jobs:
+  update-durations:
+    runs-on: ubuntu-latest
+    permissions:
+      # necessary to open PR
+      # https://github.com/peter-evans/create-pull-request#action-inputs
+      contents: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: download recent artifacts
+        run: |
+          gh run list \
+            --branch main \
+            --workflow tests \
+            --limit 10 \
+            --json databaseId \
+            --jq '.[].databaseId' \
+          | xargs \
+            -n 1 \
+          gh run download \
+            --dir ${{ runner.temp }}/artifacts/ \
+            --pattern '*-all' \
+          || true
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: combine recent durations from artifacts
+        run: python ./tools/durations/combine.py ${{ runner.temp }}/artifacts/
+
+      - name: create updated durations PR
+        uses: peter-evans/create-pull-request@v5
+        with:
+          push-to-fork: conda-bot/conda
+          token: ${{ secrets.DURATIONS_TOKEN }}
+          branch: update-durations
+          delete-branch: true
+          commit-message: Update test durations
+          author: Conda Bot <18747875+conda-bot@users.noreply.github.com>
+          committer: Conda Bot <18747875+conda-bot@users.noreply.github.com>
+          title: 🤖 Update test durations
+          body: |
+            Aggregate recent test durations for each test and update the durations file.
+
+            [durations.yml]: ${{ github.server_url }}/${{ github.repository }}/blob/main/.github/workflows/durations.yml
+
+            This PR was created automatically by the [`durations.yml`][durations.yml] workflow.

.github/workflows/tests.yml

@@ -77,28 +77,37 @@ jobs:
       fail-fast: false
       matrix:
         # test all lower versions (w/ stable conda) and upper version (w/ canary conda)
-        python-version: ['3.9', '3.10', '3.11']
-        conda-version: [release]
-        test-type: [serial, parallel]
-        include:
-          # minimum Python/conda combo
-          - python-version: '3.8'
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        conda-version: [23.5.0, release, canary]
+        test-group: [1, 2, 3]
+        exclude:
+          # testing oldest supported conda with Python 3.8
+          - python-version: '3.9'
             conda-version: 23.5.0
-            test-type: serial
-          - python-version: '3.8'
+          - python-version: '3.10'
+            conda-version: 23.5.0
+          - python-version: '3.11'
             conda-version: 23.5.0
-            test-type: parallel
-          # maximum Python/conda combo
           - python-version: '3.12'
-            conda-version: canary
-            test-type: serial
+            conda-version: 23.5.0
+          # testing stable conda with Python 3.9-3.11
+          - python-version: '3.8'
+            conda-version: release
           - python-version: '3.12'
+            conda-version: release
+          # testing canary conda with Python 3.12
+          - python-version: '3.8'
+            conda-version: canary
+          - python-version: '3.9'
+            conda-version: canary
+          - python-version: '3.10'
+            conda-version: canary
+          - python-version: '3.11'
             conda-version: canary
-            test-type: parallel
     env:
       CONDA_CHANNEL_LABEL: ${{ matrix.conda-version == 'canary' && 'conda-canary/label/dev::' || '' }}
       CONDA_VERSION: ${{ contains('canary|release', matrix.conda-version) && 'conda' || format('conda={0}', matrix.conda-version) }}
-      PYTEST_MARKER: ${{ matrix.test-type == 'serial' && 'serial' || 'not serial' }}
+      PYTEST_SPLITS: 3
 
     steps:
       - name: Checkout Source
@@ -107,7 +116,7 @@ jobs:
           fetch-depth: 0
 
       - name: Hash + Timestamp
-        run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-${{ matrix.conda-version }}-${{ matrix.test-type }}-$(date -u "+%Y%m")" >> $GITHUB_ENV
+        run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-${{ matrix.conda-version }}-${{ matrix.test-group }}-$(date -u "+%Y%m")" >> $GITHUB_ENV
 
       - name: Cache Conda
         uses: actions/cache@v4
@@ -143,8 +152,10 @@ jobs:
       - name: Run Tests
         run: pytest
           --cov=conda_build
+          --durations-path=tools/durations/${{ runner.os }}.json
+          --group=${{ matrix.test-group }}
+          --splits=${{ env.PYTEST_SPLITS }}
           -n auto
-          -m "${{ env.PYTEST_MARKER }}"
 
       - name: Upload Coverage
         uses: codecov/codecov-action@v4
@@ -158,6 +169,7 @@ jobs:
           name: test-results-${{ env.HASH }}
           path: |
             .coverage
+            tools/durations/${{ runner.os }}.json
             test-report.xml
           retention-days: 1  # temporary, combined in aggregate below
 
@@ -172,20 +184,18 @@ jobs:
       fail-fast: false
       matrix:
         # test lower version (w/ stable conda) and upper version (w/ canary conda)
-        python-version: ['3.8']
-        conda-version: [release]
-        test-type: [serial, parallel]
-        include:
-          - python-version: '3.12'
+        python-version: ['3.8', '3.12']
+        conda-version: [release, canary]
+        test-group: [1, 2, 3]
+        exclude:
+          - python-version: '3.8'
             conda-version: canary
-            test-type: serial
           - python-version: '3.12'
-            conda-version: canary
-            test-type: parallel
+            conda-version: release
     env:
       ErrorActionPreference: Stop  # powershell exit on first error
       CONDA_CHANNEL_LABEL: ${{ matrix.conda-version == 'canary' && 'conda-canary/label/dev' || 'defaults' }}
-      PYTEST_MARKER: ${{ matrix.test-type == 'serial' && 'serial' || 'not serial and not slow' }}
+      PYTEST_SPLITS: 3
 
     steps:
       - name: Checkout Source
@@ -195,7 +205,7 @@ jobs:
 
       - name: Hash + Timestamp
         shell: bash  # use bash to run date command
-        run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-${{ matrix.conda-version }}-${{ matrix.test-type }}-$(date -u "+%Y%m")" >> $GITHUB_ENV
+        run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-${{ matrix.conda-version }}-${{ matrix.test-group }}-$(date -u "+%Y%m")" >> $GITHUB_ENV
 
       - name: Cache Conda
         uses: actions/cache@v4
@@ -237,8 +247,10 @@ jobs:
         run: pytest
           --cov=conda_build
           --basetemp=${{ runner.temp }}
+          --durations-path=tools\durations\${{ runner.os }}.json
+          --group=${{ matrix.test-group }}
+          --splits=${{ env.PYTEST_SPLITS }}
           -n auto
-          -m "${{ env.PYTEST_MARKER }}"
 
       - name: Upload Coverage
         uses: codecov/codecov-action@v4
@@ -252,6 +264,7 @@ jobs:
           name: test-results-${{ env.HASH }}
           path: |
             .coverage
+            tools\durations\${{ runner.os }}.json
             test-report.xml
           retention-days: 1  # temporary, combined in aggregate below
 
@@ -270,19 +283,17 @@ jobs:
       fail-fast: false
       matrix:
         # test lower version (w/ stable conda) and upper version (w/ canary conda)
-        python-version: ['3.8']
-        conda-version: [release]
-        test-type: [serial, parallel]
-        include:
-          - python-version: '3.12'
+        python-version: ['3.8', '3.12']
+        conda-version: [release, canary]
+        test-group: [1, 2, 3]
+        exclude:
+          - python-version: '3.8'
             conda-version: canary
-            test-type: serial
           - python-version: '3.12'
-            conda-version: canary
-            test-type: parallel
+            conda-version: release
     env:
       CONDA_CHANNEL_LABEL: ${{ matrix.conda-version == 'canary' && 'conda-canary/label/dev' || 'defaults' }}
-      PYTEST_MARKER: ${{ matrix.test-type == 'serial' && 'serial' || 'not serial' }}
+      PYTEST_SPLITS: 3
 
     steps:
       - name: Checkout Source
@@ -291,7 +302,7 @@ jobs:
           fetch-depth: 0
 
       - name: Hash + Timestamp
-        run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-${{ matrix.conda-version }}-${{ matrix.test-type }}-$(date -u "+%Y%m")" >> $GITHUB_ENV
+        run: echo "HASH=${{ runner.os }}-${{ runner.arch }}-Py${{ matrix.python-version }}-${{ matrix.conda-version }}-${{ matrix.test-group }}-$(date -u "+%Y%m")" >> $GITHUB_ENV
 
       - name: Cache Conda
         uses: actions/cache@v4
@@ -330,8 +341,10 @@ jobs:
       - name: Run Tests
         run: pytest
           --cov=conda_build
+          --durations-path=tools/durations/${{ runner.os }}.json
+          --group=${{ matrix.test-group }}
+          --splits=${{ env.PYTEST_SPLITS }}
           -n auto
-          -m "${{ env.PYTEST_MARKER }}"
 
       - name: Upload Coverage
         uses: codecov/codecov-action@v4
@@ -345,6 +358,7 @@ jobs:
           name: test-results-${{ env.HASH }}
           path: |
             .coverage
+            tools/durations/${{ runner.os }}.json
             test-report.xml
           retention-days: 1  # temporary, combined in aggregate below
 

tests/cli/test_main_render.py

@@ -1,7 +1,10 @@
 # Copyright (C) 2014 Anaconda, Inc
 # SPDX-License-Identifier: BSD-3-Clause
+from __future__ import annotations
+
 import os
 import sys
+from typing import TYPE_CHECKING
 
 import pytest
 import yaml
@@ -12,6 +15,12 @@
 
 from ..utils import metadata_dir
 
+if TYPE_CHECKING:
+    from pytest import CaptureFixture
+
+    from conda_build.config import Config
+    from conda_build.metadata import MetaData
+
 
 def test_render_add_channel():
     """This recipe requires the conda_build_test_requirement package, which is
@@ -67,7 +76,10 @@ def test_render_without_channel_fails(tmp_path):
 
 
 def test_render_output_build_path(
-    testing_workdir, testing_config, testing_metadata, capfd, caplog
+    testing_workdir: str,
+    testing_config: Config,
+    testing_metadata: MetaData,
+    capfd: CaptureFixture,
 ):
     api.output_yaml(testing_metadata, "meta.yaml")
     args = ["--output", testing_workdir]
@@ -83,7 +95,10 @@ def test_render_output_build_path(
 
 
 def test_render_output_build_path_and_file(
-    testing_workdir, testing_config, testing_metadata, capfd, caplog
+    testing_workdir: str,
+    testing_config: Config,
+    testing_metadata: MetaData,
+    capfd: CaptureFixture,
 ):
     api.output_yaml(testing_metadata, "meta.yaml")
     rendered_filename = "out.yaml"

tests/conftest.py

@@ -55,16 +55,12 @@ def testing_workdir(monkeypatch: MonkeyPatch, tmp_path: Path) -> Iterator[str]:
 
 
 @pytest.fixture(scope="function")
-def testing_homedir() -> Iterator[Path]:
+def testing_homedir(monkeypatch: MonkeyPatch) -> Iterator[Path]:
     """Create a temporary testing directory in the users home directory; cd into dir before test, cd out after."""
-    saved = Path.cwd()
     try:
         with tempfile.TemporaryDirectory(dir=Path.home(), prefix=".pytest_") as home:
-            os.chdir(home)
-
-            yield home
-
-            os.chdir(saved)
+            monkeypatch.chdir(home)
+            yield Path(home)
     except OSError:
         pytest.xfail(
             f"failed to create temporary directory () in {'%HOME%' if on_win else '${HOME}'} "
@@ -73,7 +69,7 @@ def testing_homedir() -> Iterator[Path]:
 
 
 @pytest.fixture(scope="function")
-def testing_config(testing_workdir):
+def testing_config(testing_workdir: str) -> Config:
     def boolify(v):
         return v == "true"
 

tests/requirements-ci.txt

@@ -1,4 +1,5 @@
 anaconda-client
+conda-forge::pytest-split
 conda-forge::xdoctest
 conda-verify
 contextlib2