From a872855e34f645344abc90c2df2d582ada1ad5d5 Mon Sep 17 00:00:00 2001 From: roomrys Date: Wed, 11 Sep 2024 20:46:59 -0700 Subject: [PATCH] Consolidate workflow and separate build and upload jobs --- .github/workflows/build_tensorflow.yml | 127 ++++++++++++++----------- 1 file changed, 73 insertions(+), 54 deletions(-) diff --git a/.github/workflows/build_tensorflow.yml b/.github/workflows/build_tensorflow.yml index 61f2e45..b7b1adb 100644 --- a/.github/workflows/build_tensorflow.yml +++ b/.github/workflows/build_tensorflow.yml @@ -14,36 +14,35 @@ on: - "requirements.tensorflow.txt" - "requirements.tensorflow_macos.txt" +# If RUN_BUILD_JOB is set to true, then RUN_ID will be overwritten to the current run id +env: + RUN_BUILD_JOB: true + RUN_ID: 10713717594 # Only used if RUN_BUILD_JOB is false (to dowload build artifact) + jobs: build: name: Build package (${{ matrix.os }}) runs-on: ${{ matrix.os }} strategy: - fail-fast: false + fail-fast: false # TODO(LM): Set to true matrix: - os: ['ubuntu-22.04', 'windows-2022', 'macos-14'] + os: ["ubuntu-22.04", "windows-2022", "macos-14"] + include: + - os-folder: win-64 + - build-folder: .conda.tensorflow + - os: "ubuntu-22.04" + os-folder: linux-64 + - os: "macos-14" + os-folder: osx-arm64 + build-folder: .conda.tensorflow_macos steps: # Checkout the repository - name: Checkout + if: env.RUN_BUILD_JOB == 'true' uses: actions/checkout@v4 - - name: Make conda prefix directory (Windows) - if: matrix.os == 'windows-2022' - shell: powershell - run: | - New-Item -ItemType Directory -Force -Path C:\c - - # https://github.com/conda-incubator/setup-miniconda?tab=readme-ov-file#use-a-different-environment-name-or-path - - name: Setup Miniconda (Windows) - if: matrix.os == 'windows-2022' - uses: conda-incubator/setup-miniconda@v3.0.3 - with: - python-version: "3.10" - environment-file: environment.tensorflow.yml - activate-environment: C:\c\tf - - - name: Setup Miniconda (not Windows) - if: matrix.os != 'windows-2022' + - name: Setup Miniconda + if: env.RUN_BUILD_JOB == 'true' uses: conda-incubator/setup-miniconda@v3.0.3 with: python-version: "3.10" @@ -51,57 +50,77 @@ jobs: activate-environment: tf - name: Print environment info + if: env.RUN_BUILD_JOB == 'true' shell: bash -l {0} run: | which python conda info conda list - - name: Build conda package (Windows) - if: matrix.os == 'windows-2022' - shell: powershell - run: | - conda build --debug .conda.tensorflow --output-folder C:\c\build.tensorflow -c conda-forge - - - name: Build conda package (Ubuntu) - if: matrix.os == 'ubuntu-22.04' + - name: Build conda package + if: env.RUN_BUILD_JOB == 'true' shell: bash -l {0} run: | - conda build --debug .conda.tensorflow --output-folder build.tensorflow -c conda-forge + conda build --debug ${{ matrix.build-folder }} --output-folder build.tensorflow -c conda-forge - # https://github.com/anaconda/anaconda-client/issues/529 - - name: Upload to Anaconda (Windows) - if: matrix.os == 'windows-2022' - env: - ANACONDA_PASSWORD: ${{ secrets.ANACONDA_PASSWORD }} - shell: powershell - run: | - anaconda login --username sleap-deps --password "$ANACONDA_PASSWORD" - anaconda -v upload "C:\c\build.tensorflow\win-64\*.tar.bz2" --label dev --user sleap-deps - anaconda logout + # Upload the build artifact incase anaconda upload fails + - name: Upload conda package artifact + if: env.RUN_BUILD_JOB == 'true' + uses: actions/upload-artifact@v4 + with: + name: tensorflow-build-${{ matrix.os-folder }} + path: build.tensorflow # Upload entire build directory + retention-days: 1 - - name: Upload to Anaconda (Ubuntu) - if: matrix.os == 'ubuntu-22.04' - env: - ANACONDA_PASSWORD: ${{ secrets.ANACONDA_PASSWORD }} - shell: bash -l {0} - run: | - anaconda login --username sleap-deps --password "$ANACONDA_PASSWORD" - anaconda -v upload build.tensorflow/linux-64/*.tar.bz2 --label dev --user sleap-deps - anaconda logout + upload: + name: Upload package (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false # TODO(LM): Set to true + matrix: + os: ["ubuntu-22.04", "windows-2022", "macos-14"] + include: + - os-folder: win-64 + - os: "ubuntu-22.04" + os-folder: linux-64 + - os: "macos-14" + os-folder: osx-arm64 - - name: Build conda package (MacOS) - if: matrix.os == 'macos-14' + steps: + - name: Use current run id for conda package download shell: bash -l {0} - run: | - conda build --debug .conda.tensorflow_macos --output-folder build.tensorflow_macos -c conda-forge + if: env.RUN_BUILD_JOB == 'true' + run: echo "RUN_ID=${{ github.run_id }}" >> $GITHUB_ENV + + # https://github.com/actions/download-artifact?tab=readme-ov-file#usage + - name: Download conda package artifact + uses: actions/download-artifact@v4 + id: download + with: + name: sleap-build-${{ matrix.os-folder }} + path: build.tensorflow + run-id: ${{ env.RUN_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.pyver }} + + - name: Setup Miniconda + if: env.RUN_BUILD_JOB == 'true' + uses: conda-incubator/setup-miniconda@v3.0.3 + with: + python-version: "3.10" + environment-file: environment.tensorflow.yml + activate-environment: tf - - name: Upload to Anaconda (MacOS) - if: matrix.os == 'macos-14' + - name: Upload to Anaconda + if: env.RUN_BUILD_JOB == 'true' env: ANACONDA_PASSWORD: ${{ secrets.ANACONDA_PASSWORD }} shell: bash -l {0} run: | anaconda login --username sleap-deps --password "$ANACONDA_PASSWORD" - anaconda -v upload build.tensorflow_macos/osx-arm64/*.tar.bz2 --label dev --user sleap-deps + anaconda -v upload build.tensorflow/${{ matrix.os-folder }}/*.tar.bz2 --label dev --user sleap-deps anaconda logout