Fix CI on macosx-arm64 (#1734)

* Build conda package locally * Try 2.8.4 * Merge develop into branch to fix dependencies * Change tensorflow version to 2.7.4 in where conda packages are used * Make tensorflow requirements in pypi looser * Conda package has TensorFlow 2.7.0 and h5py and numpy installed via conda * Change tensorflow version in `environment_no_cuda.yml` to test using CI * Test new sleap/tensorflow package * Reset build number * Bump version * Update mac deps * Update to Arm64 Mac runners * pin `importlib-metadata` * Pin more stuff on mac * constrain `opencv` version due to new qt dependencies * Update more mac stuff * Patches to get to green * More mac skipping * Re-enable mac tests * Handle GPU re-init * Fix mac build CI * Widen tolerance for movenet correctness test * Fix build ci * Try for manual build without upload * Try to reduce training CI time * Rework actions * Fix miniforge usage * Tweaks * Fix build ci * Disable manual build * Try merging CI coverage * GPU/CPU usage in tests * Lint * Clean up * Fix test skip condition * Remove scratch test --------- Co-authored-by: eberrigan <berri104@gmail.com>
talmolab · Apr 8, 2024 · d4ad3bb · d4ad3bb
1 parent 7ed1229
commit d4ad3bb
Show file tree

Hide file tree

Showing 35 changed files with 182 additions and 161 deletions.
diff --git a/.conda/condarc.yaml b/.conda/condarc.yaml
@@ -1,6 +1,6 @@
 channels:
   - conda-forge
   - nvidia
-  # - https://conda.anaconda.org/sleap/label/dev
+  - https://conda.anaconda.org/sleap/label/dev
   - sleap
   - anaconda
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -13,8 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # os: ["ubuntu-22.04", "windows-2022", "macos-14"]
-        os: ["ubuntu-22.04", "windows-2022"]  # TODO: Fix mac tests
+        os: ["ubuntu-22.04", "windows-2022", "macos-14"]
         # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategymatrixinclude
         include:
           # Use this condarc as default

diff --git a/.github/workflows/build_ci.yml b/.github/workflows/build_ci.yml
@@ -26,6 +26,7 @@ jobs:
           # Use this condarc as default
           - condarc: .conda/condarc.yaml
           - wheel_name: sleap-wheel-linux
+          - pyver: "3.7"
     steps:
       # Setup
       - uses: actions/checkout@v2
@@ -39,15 +40,16 @@ jobs:
           path: ~/conda_pkgs_dir
           key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment_build.yml', 'pyproject.toml') }}
 
-      - name: Setup Miniconda for Build
-        # https://github.com/conda-incubator/setup-miniconda
-        uses: conda-incubator/setup-miniconda@v2.0.1
+      - name: Setup Miniconda
+        uses: conda-incubator/setup-miniconda@v3.0.3
         with:
-          python-version: 3.7
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
-          environment-file: environment_build.yml
+          miniforge-version: latest
           condarc-file: ${{ matrix.condarc }}
+          python-version: ${{ matrix.pyver }}
+          environment-file: environment_build.yml
           activate-environment: sleap_ci
+          conda-solver: "libmamba"
+          # use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
 
       - name: Print build environment info
         shell: bash -l {0}
@@ -77,8 +79,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # os: ["ubuntu-22.04", "windows-2022", "macos-14"]
-        os: ["ubuntu-22.04", "windows-2022"]
+        os: ["ubuntu-22.04", "windows-2022", "macos-14"]
         # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategymatrixinclude
         include:
           # Default values
@@ -90,9 +91,11 @@ jobs:
               pip install '$wheel_path'[dev]
           - test_args: pytest --durations=-1 tests/
           - condarc: .conda/condarc.yaml
+          - pyver: "3.7"
           # Use special condarc if macos
-          # - os: "macos-14"
-          #   condarc: .conda_mac/condarc.yaml
+          - os: "macos-14"
+            condarc: .conda_mac/condarc.yaml
+            pyver: "3.10"
           # Ubuntu specific values
           - os: ubuntu-22.04
             # Otherwise core dumped in github actions
@@ -109,10 +112,10 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v3
 
-      - name: Set up Python 3.7
+      - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: ${{ matrix.pyver }}
 
       # Download wheel
       - name: Download wheel artifact

diff --git a/.github/workflows/build_manual.yml b/.github/workflows/build_manual.yml
@@ -8,9 +8,11 @@ on:
     paths:
       - '.conda/meta.yaml'
       - '.conda_mac/meta.yaml'
+      - '.github/workflows/build_manual.yml'
     branches:
       # - develop
       - fakebranch
+      # - talmo/fix-mac-v140
 
 jobs:
   build:
@@ -24,9 +26,11 @@ jobs:
         include:
           # Use this condarc as default
           - condarc: .conda/condarc.yaml
+          - pyver: "3.7"
           # Use special condarc if macos
           - os: "macos-14"
             condarc: .conda_mac/condarc.yaml
+            pyver: "3.9"
     steps:
       # Setup
       - uses: actions/checkout@v2
@@ -38,20 +42,24 @@ jobs:
         with:
           path: ~/conda_pkgs_dir
           key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment_build.yml', 'requirements.txt') }}
+
       - name: Setup Miniconda
-        # https://github.com/conda-incubator/setup-miniconda
-        uses: conda-incubator/setup-miniconda@v2.0.1
+        uses: conda-incubator/setup-miniconda@v3.0.3
         with:
-          python-version: 3.7
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
-          environment-file: environment_build.yml
+          miniforge-version: latest
           condarc-file: ${{ matrix.condarc }}
+          python-version: ${{ matrix.pyver }}
+          environment-file: environment_build.yml
           activate-environment: sleap_ci
+          conda-solver: "libmamba"
+          # use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+
       - name: Print environment info
         shell: bash -l {0}
         run: |
           which python
           conda info
+          conda list
 
       # Build pip wheel (Not Windows)
       - name: Build pip wheel (Not Windows)
@@ -90,55 +98,55 @@ jobs:
         run: |
           conda build .conda_mac --output-folder build
 
-      # Login to conda (Ubuntu)
-      - name: Login to Anaconda (Ubuntu)
-        if: matrix.os == 'ubuntu-22.04'
-        env:
-          ANACONDA_LOGIN: ${{ secrets.ANACONDA_LOGIN }}
-        shell: bash -l {0}
-        run: |
-          yes 2>/dev/null | anaconda login --username sleap --password "$ANACONDA_LOGIN" || true
+      # # Login to conda (Ubuntu)
+      # - name: Login to Anaconda (Ubuntu)
+      #   if: matrix.os == 'ubuntu-22.04'
+      #   env:
+      #     ANACONDA_LOGIN: ${{ secrets.ANACONDA_LOGIN }}
+      #   shell: bash -l {0}
+      #   run: |
+      #     yes 2>/dev/null | anaconda login --username sleap --password "$ANACONDA_LOGIN" || true
 
-      # Login to conda (Windows)
-      - name: Login to Anaconda (Windows)
-        if: matrix.os == 'windows-2022'
-        env:
-          ANACONDA_LOGIN: ${{ secrets.ANACONDA_LOGIN }}
-        shell: powershell
-        run: |
-          echo "yes" | anaconda login --username sleap --password "$env:ANACONDA_LOGIN"
+      # # Login to conda (Windows)
+      # - name: Login to Anaconda (Windows)
+      #   if: matrix.os == 'windows-2022'
+      #   env:
+      #     ANACONDA_LOGIN: ${{ secrets.ANACONDA_LOGIN }}
+      #   shell: powershell
+      #   run: |
+      #     echo "yes" | anaconda login --username sleap --password "$env:ANACONDA_LOGIN"
 
-      # Login to conda (Mac)
-      - name: Login to Anaconda (Mac)
-        if: matrix.os == 'macos-14'
-        env:
-          ANACONDA_LOGIN: ${{ secrets.ANACONDA_LOGIN }}
-        shell: bash -l {0}
-        run: |
-          yes 2>/dev/null | anaconda login --username sleap --password "$ANACONDA_LOGIN" || true
+      # # Login to conda (Mac)
+      # - name: Login to Anaconda (Mac)
+      #   if: matrix.os == 'macos-14'
+      #   env:
+      #     ANACONDA_LOGIN: ${{ secrets.ANACONDA_LOGIN }}
+      #   shell: bash -l {0}
+      #   run: |
+      #     yes 2>/dev/null | anaconda login --username sleap --password "$ANACONDA_LOGIN" || true
 
-      # Upload conda package (Windows)
-      - name: Upload conda package (Windows/dev)
-        if: matrix.os == 'windows-2022'
-        shell: powershell
-        run: |
-          anaconda -v upload "build\win-64\*.tar.bz2" --label dev
+      # # Upload conda package (Windows)
+      # - name: Upload conda package (Windows/dev)
+      #   if: matrix.os == 'windows-2022'
+      #   shell: powershell
+      #   run: |
+      #     anaconda -v upload "build\win-64\*.tar.bz2" --label dev
 
-      # Upload conda package (Ubuntu)
-      - name: Upload conda package (Ubuntu/dev)
-        if: matrix.os == 'ubuntu-22.04'
-        shell: bash -l {0}
-        run: |
-          anaconda -v upload build/linux-64/*.tar.bz2 --label dev
+      # # Upload conda package (Ubuntu)
+      # - name: Upload conda package (Ubuntu/dev)
+      #   if: matrix.os == 'ubuntu-22.04'
+      #   shell: bash -l {0}
+      #   run: |
+      #     anaconda -v upload build/linux-64/*.tar.bz2 --label dev
 
-      # Upload conda package (Mac)
-      - name: Upload conda package (Mac/dev)
-        if: matrix.os == 'macos-14'
-        shell: bash -l {0}
-        run: |
-          anaconda -v upload build/osx-arm64/*.tar.bz2 --label dev
+      # # Upload conda package (Mac)
+      # - name: Upload conda package (Mac/dev)
+      #   if: matrix.os == 'macos-14'
+      #   shell: bash -l {0}
+      #   run: |
+      #     anaconda -v upload build/osx-arm64/*.tar.bz2 --label dev
 
-      - name: Logout from Anaconda
-        shell: bash -l {0}
-        run: |
-          anaconda logout
+      # - name: Logout from Anaconda
+      #   shell: bash -l {0}
+      #   run: |
+      #     anaconda logout
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -11,37 +11,20 @@ on:
       - "environment_no_cuda.yml"
       - "requirements.txt"
       - "dev_requirements.txt"
-  push:
-    branches:
-      - master
-      - develop
-    paths:
-      - "sleap/**"
-      - "tests/**"
-      - ".github/workflows/ci.yml"
-      - "environment_no_cuda.yml"
-      - "requirements.txt"
-      - "dev_requirements.txt"
+  # push:
+  #   branches:
+  #     - main
+  #     - develop
+  #   paths:
+  #     - "sleap/**"
+  #     - "tests/**"
+  #     - ".github/workflows/ci.yml"
+  #     - "environment_no_cuda.yml"
+  #     - "requirements.txt"
+  #     - "dev_requirements.txt"
 
 jobs:
-  type_check:
-    name: Type Check
-    runs-on: "ubuntu-22.04"
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v3
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.7
-      - name: Install Dependencies
-        run: |
-          pip install mypy
-      - name: Run MyPy
-        # TODO: remove this once all MyPy errors get fixed
-        continue-on-error: true
-        run: |
-          mypy --follow-imports=skip --ignore-missing-imports sleap tests
+  # Lint
   lint:
     name: Lint
     runs-on: "ubuntu-22.04"
@@ -59,59 +42,52 @@ jobs:
       - name: Run Black
         run: |
           black --check sleap tests
+
+  # Tests
   tests:
     name: Tests (${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        # os: ["ubuntu-22.04", "windows-2022", "macos-14"]  # TODO: Fix Mac tests
-        os: ["ubuntu-22.04", "windows-2022"]
+        os: ["ubuntu-22.04", "windows-2022", "macos-14"]
         include:
           # Default values
           - env_file: environment_no_cuda.yml
-          - test_args: --durations=-1 tests/
           # Mac specific values
-          # - os: macos-14
-          #   env_file: environment_mac.yml
-          # Ubuntu specific values
-          - os: ubuntu-22.04
-            test_args: --cov=sleap --cov-report=xml --durations=-1 tests/
+          - os: macos-14
+            env_file: environment_mac.yml
+
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
 
-      - name: Setup Micromamba
-        # https://github.com/mamba-org/setup-micromamba
-        uses: mamba-org/setup-micromamba@v1
+      - name: Setup Conda
+        uses: conda-incubator/setup-miniconda@v3.0.3
         with:
-          micromamba-version: '1.4.6-0'
+          miniforge-version: latest
+          conda-solver: "libmamba"
           environment-file: ${{ matrix.env_file }}
-          environment-name: sleap_ci
-          init-shell: >-
-            bash
-            powershell
-          post-cleanup: all
+          activate-environment: sleap_ci
 
       # Print environment info
       - name: Print environment info
         shell: bash -l {0}
         run: |
           which python
-          micromamba info
-          micromamba list
+          conda info
+          conda list
           pip freeze
       
       # Test environment
       - name: Test with pytest
         shell: bash -l {0}
         run: |
-          pytest ${{ matrix.test_args }}
+          pytest --cov=sleap --cov-report=xml --durations=-1 tests/
 
       # Upload coverage
       - name: Upload coverage
-        uses: codecov/codecov-action@v1
-        if: matrix.os == 'ubuntu-22.04'
+        uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: true