diff --git a/.conda.tensorflow/bld.bat b/.conda.tensorflow/bld.bat new file mode 100644 index 0000000..75ef64d --- /dev/null +++ b/.conda.tensorflow/bld.bat @@ -0,0 +1,7 @@ +@echo off + +set PIP_NO_INDEX=False +set PIP_NO_DEPENDENCIES=False +set PIP_IGNORE_INSTALLED=False + +pip install --no-cache-dir -r .\requirements.tensorflow.txt \ No newline at end of file diff --git a/.conda.tensorflow/build.sh b/.conda.tensorflow/build.sh new file mode 100644 index 0000000..775d217 --- /dev/null +++ b/.conda.tensorflow/build.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +export PIP_NO_INDEX=False +export PIP_NO_DEPENDENCIES=False +export PIP_IGNORE_INSTALLED=False + +pip install --no-cache-dir -r ./requirements.tensorflow.txt + +# Copy the activate scripts to $PREFIX/etc/conda/activate.d. +# This will allow them to be run on environment activation. +for CHANGE in "activate" "deactivate" +do + mkdir -p "${PREFIX}/etc/conda/${CHANGE}.d" + cp "${RECIPE_DIR}/tensorflow_${CHANGE}.sh" "${PREFIX}/etc/conda/${CHANGE}.d/tensorflow_${CHANGE}.sh" +done \ No newline at end of file diff --git a/.conda.tensorflow/meta.yaml b/.conda.tensorflow/meta.yaml new file mode 100644 index 0000000..81818cf --- /dev/null +++ b/.conda.tensorflow/meta.yaml @@ -0,0 +1,51 @@ +# Ref: https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html + +package: + name: tensorflow + version: 2.9.2 + +about: + home: https://tensorflow.org + license: Apache License + summary: 'TensorFlow conda package based on the PyPI wheels. + + For GPU support, install cudatoolkit 11.3.1 and cudnn 8.2.1 which are available as conda packages on the default channel.' + +build: + number: 0 + +source: + path: ../ + +requirements: + build: + - '{{ compiler("c") }}' + - '{{ compiler("cxx") }}' + + host: + - python>=3.10.0,<3.11.0 + - numpy + - h5py + - protobuf + - certifi + - importlib-metadata + - six + - typing-extensions + - zipp + - pip + + run: + - python>=3.10.0,<3.11.0 + - numpy + - h5py + - protobuf + - certifi + - importlib-metadata + - six + - typing-extensions + - zipp + - pip + +test: + imports: + - tensorflow \ No newline at end of file diff --git a/.conda.tensorflow/tensorflow_activate.sh b/.conda.tensorflow/tensorflow_activate.sh new file mode 100644 index 0000000..885879a --- /dev/null +++ b/.conda.tensorflow/tensorflow_activate.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +# Remember the old library path for when we deactivate +export SLEAP_OLD_LD_LIBRARY_PATH=$LD_LIBRARY_PATH +# Help CUDA find GPUs! +export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH \ No newline at end of file diff --git a/.conda.tensorflow/tensorflow_deactivate.sh b/.conda.tensorflow/tensorflow_deactivate.sh new file mode 100644 index 0000000..857c0f4 --- /dev/null +++ b/.conda.tensorflow/tensorflow_deactivate.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +# Reset to the old library path for when deactivating the environment +export LD_LIBRARY_PATH=$SLEAP_OLD_LD_LIBRARY_PATH \ No newline at end of file diff --git a/.github/workflows/build_tensorflow.yml b/.github/workflows/build_tensorflow.yml index c4489b8..b105bf1 100644 --- a/.github/workflows/build_tensorflow.yml +++ b/.github/workflows/build_tensorflow.yml @@ -4,54 +4,156 @@ on: push: branches: - main - - elizabeth/add-tensorflow-macos-conda-package + - elizabeth/add-windows-linux-tensorflow-conda-package paths: - ".github/workflows/build_tensorflow.yml" + - ".conda.tensorflow/**" - ".conda.tensorflow_macos/**" - "environment.tensorflow.yml" + - "requirements.tensorflow.txt" - "requirements.tensorflow_macos.txt" +# If RUN_BUILD_JOB is set to true, then RUN_ID will be overwritten to the current run id +env: + RUN_BUILD_JOB: false # TODO(LM): Set to true + RUN_ID: 10824742361 # Only used if RUN_BUILD_JOB is false (to dowload build artifact) + jobs: build: name: Build package (${{ matrix.os }}) runs-on: ${{ matrix.os }} strategy: - fail-fast: false + fail-fast: true matrix: - os: ["macos-14"] + os: ["ubuntu-22.04", "windows-2022", "macos-14"] + include: + - os-folder: linux-64 + - build-folder: .conda.tensorflow + - os: "windows-2022" + os-folder: win-64 + env-prefix: C:\c\ + - os: "macos-14" + os-folder: osx-arm64 + build-folder: .conda.tensorflow_macos steps: # Checkout the repository - name: Checkout + if: env.RUN_BUILD_JOB == 'true' uses: actions/checkout@v4 - - name: Setup Miniconda (Ubuntu and MacOS) - if: matrix.os == 'macos-14' + - name: Make conda prefix directory (Windows) + if: matrix.os == 'windows-2022' + shell: powershell + run: | + New-Item -ItemType Directory -Force -Path ${{ matrix.env-prefix }} + + - name: Setup Miniconda + if: env.RUN_BUILD_JOB == 'true' uses: conda-incubator/setup-miniconda@v3.0.3 with: python-version: "3.10" environment-file: environment.tensorflow.yml - activate-environment: tf + activate-environment: ${{ matrix.env-prefix }}tf - name: Print environment info + if: env.RUN_BUILD_JOB == 'true' shell: bash -l {0} run: | which python conda info conda list - - name: Build conda package (MacOS) - if: matrix.os == 'macos-14' + - name: Build conda package (Windows) + if: (runner.os == 'Windows') && (env.RUN_BUILD_JOB == 'true') + shell: powershell + run: | + conda build --debug ${{ matrix.build-folder }} --output-folder ${{ matrix.env-prefix }}build -c conda-forge + + - name: Build conda package (not Windows) + if: (runner.os != 'Windows') && (env.RUN_BUILD_JOB == 'true') shell: bash -l {0} run: | - conda build --debug .conda.tensorflow_macos --output-folder build.tensorflow_macos -c conda-forge + conda build --debug ${{ matrix.build-folder }} --output-folder build -c conda-forge + + # Upload the build artifact incase anaconda upload fails + - name: Upload conda package artifact + if: env.RUN_BUILD_JOB == 'true' + uses: actions/upload-artifact@v4 + with: + name: tensorflow-build-${{ matrix.os-folder }} + path: ${{ matrix.env-prefix }}build # Upload entire build directory + retention-days: 1 + + upload: + name: Upload package (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + needs: build + strategy: + fail-fast: false # TODO(LM): Set to true + matrix: + os: ["ubuntu-22.04", "windows-2022", "macos-14"] + include: + - os-folder: win-64 + - os: "ubuntu-22.04" + os-folder: linux-64 + - os: "macos-14" + os-folder: osx-arm64 + + steps: + # Checkout the repository + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.pyver }} + + - name: Setup Miniconda + uses: conda-incubator/setup-miniconda@v3.0.3 + with: + python-version: "3.10" + environment-file: environment.tensorflow.yml + activate-environment: tf + + - name: Use current run id for conda package download + shell: bash -l {0} + if: env.RUN_BUILD_JOB == 'true' + run: echo "RUN_ID=${{ github.run_id }}" >> $GITHUB_ENV + + # https://github.com/actions/download-artifact?tab=readme-ov-file#usage + - name: Download conda package artifact + uses: actions/download-artifact@v4 + id: download + with: + name: tensorflow-build-${{ matrix.os-folder }} + path: build + run-id: ${{ env.RUN_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: List items in current directory + shell: bash -l {0} + run: | + ls . + ls -R build + + - name: Upload to Anaconda (Windows) + if: runner.os == 'Windows' + env: + ANACONDA_PASSWORD: ${{ secrets.ANACONDA_PASSWORD }} + shell: bash -l {0} + run: | + anaconda login --username sleap-deps --password "$ANACONDA_PASSWORD" + anaconda -v upload "build/${{ matrix.os-folder }}/*.tar.bz2" --label dev --user sleap-deps + anaconda logout - - name: Upload to Anaconda (MacOS) - if: matrix.os == 'macos-14' + - name: Upload to Anaconda (not Windows) + if: runner.os != 'Windows' env: ANACONDA_PASSWORD: ${{ secrets.ANACONDA_PASSWORD }} shell: bash -l {0} run: | anaconda login --username sleap-deps --password "$ANACONDA_PASSWORD" - anaconda -v upload build.tensorflow_macos/osx-arm64/*.tar.bz2 --label dev --user sleap-deps + anaconda -v upload build/${{ matrix.os-folder }}/*.tar.bz2 --label dev --user sleap-deps anaconda logout diff --git a/README.md b/README.md index ae060ea..d02739c 100644 --- a/README.md +++ b/README.md @@ -7,28 +7,32 @@ There is a Github Actions workflow `build_tensorflow.yml` that uses Github runne 1. Create the tensorflow build environment: -```bash -mamba env create -f environment.tensorflow.yml -n tf -``` + ```bash + mamba env create -f environment.tensorflow.yml -n tf + ``` 2. Activate the tensorflow build environment: -```bash -mamba activate tf -``` + ```bash + mamba activate tf + ``` 3. Build the tensorflow conda package: -#### Mac + #### Windows or Linux -```bash -conda build --debug.conda.tensorflow_macos --output-folder build.tensorflow_macos -c conda-forge -``` + ```bash + conda build --debug .conda.tensorflow --output-folder build.tensorflow -c conda-forge + ``` -4. Test the conda package: + #### Mac + + ```bash + conda build --debug .conda.tensorflow_macos --output-folder build.tensorflow -c conda-forge + ``` -#### Mac +4. Test the conda package: -```bash -mamba create -n tf_macos -c ./build.tensorflow_macos -c conda-forge tensorflow -``` + ```bash + mamba create -n tf_build -c ./build.tensorflow -c conda-forge tensorflow + ``` diff --git a/requirements.tensorflow.txt b/requirements.tensorflow.txt new file mode 100644 index 0000000..eeb92e4 --- /dev/null +++ b/requirements.tensorflow.txt @@ -0,0 +1,4 @@ +# tensorflow==2.10.1 # https://github.com/talmolab/sleap/issues/1721 +# tensorflow==2.7.0 # 2.7.0 worked in the past +# tensorflow==2.9.0 # Trying to match the version of tensorflow-macos filename too long? +tensorflow==2.9.2 # 2.9.2 is the latest version in the 2.9 series \ No newline at end of file