[WIP] github actions migration #78
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Install and test | |
| on: | |
| pull_request: {} | |
| push: | |
| branches: | |
| - main | |
| tags: [ "v*" ] | |
| schedule: | |
| - cron: "0 5 * * *" | |
| #this is 9PM PST | |
| jobs: | |
| pre-commit: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4.1.1 | |
| - name: Setup python | |
| uses: actions/setup-python@v5.0.0 | |
| with: | |
| python-version: '3.9.16' | |
| - name: install dependencies | |
| run: |- | |
| pip install -U pip setuptools pre-commit | |
| # Install the hooks now so that they'll be cached | |
| pre-commit install-hooks | |
| - name: Check Code Style using pre-commit | |
| run: |- | |
| SKIP=eslint pre-commit run --show-diff-on-failure --all-files | |
| python_lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4.1.1 | |
| - name: Setup python | |
| uses: actions/setup-python@v5.0.0 | |
| with: | |
| python-version: '3.9.16' | |
| - name: setup | |
| run: |- | |
| pip install -U pip | |
| pip install -U --prefer-binary \ | |
| black==23.1.0 \ | |
| flake8 \ | |
| flake8-bugbear==22.6.22 \ | |
| flake8-builtins \ | |
| flake8-comprehensions \ | |
| flake8-return \ | |
| flake8-simplify \ | |
| hypothesis==6.29.3 \ | |
| isort==5.12.0 \ | |
| mypy \ | |
| numpy==1.26.4 \ | |
| pytest \ | |
| sphinx \ | |
| tqdm | |
| pip install --prefer-binary -r requirements.txt torch --progress-bar off | |
| - name: run black | |
| run: |- | |
| black --version | |
| black --exclude '/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|_build|buck-out|build|dist)|examples/tutorials/(notebooks|nb_python)' src_python/habitat_sim/. examples/. tests/. setup.py --diff | |
| black --exclude '/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|_build|buck-out|build|dist)|examples/tutorials/(notebooks|nb_python)' src_python/habitat_sim/. examples/. tests/. setup.py --check | |
| - name: run isort | |
| run: |- | |
| isort --version | |
| isort src_python/habitat_sim/. examples/. tests/. setup.py --diff | |
| isort src_python/habitat_sim/. examples/. tests/. setup.py --check-only | |
| - name: run flake8 | |
| run: |- | |
| flake8 --version | |
| flake8 src_python/habitat_sim/. examples/. tests/. setup.py | |
| - name: run mypy | |
| run: mypy | |
| install_and_test_ubuntu: | |
| runs-on: 4-core-ubuntu-gpu-t4 | |
| env: | |
| FPS_THRESHOLD: 900 | |
| defaults: | |
| run: | |
| shell: bash -el {0} | |
| steps: | |
| - uses: actions/checkout@v4.1.1 | |
| with: | |
| path: "./habitat-sim" | |
| - name: CPU info | |
| run: cat /proc/cpuinfo | |
| - uses: "./habitat-sim/.github/actions/install_ubuntu_deps" | |
| - uses: "./habitat-sim/.github/actions/install_ubuntu_gpu_deps" | |
| - name: Build, install habitat-sim | |
| run: |- | |
| #give cmake ownership to the runner for installation | |
| sudo chown runner -R /opt/cmake312/ | |
| #activate conda env | |
| export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH | |
| conda init | |
| source ~/.bashrc | |
| conda activate habitat | |
| #install habitat-sim | |
| cd habitat-sim | |
| pip install -r requirements.txt --progress-bar off | |
| git submodule update --init --recursive --jobs 8 | |
| python -u setup.py install --build-type "Release" --lto --headless --bullet | |
| - name: Download test data | |
| run: |- | |
| # Disable clone protection for git lfs | |
| export GIT_CLONE_PROTECTION_ACTIVE=false | |
| git --version | |
| git-lfs --version | |
| export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH | |
| conda init | |
| source ~/.bashrc | |
| conda activate habitat | |
| conda install -y gitpython git-lfs | |
| cd habitat-sim | |
| git lfs install | |
| python src_python/habitat_sim/utils/datasets_download.py --uids ci_test_assets ycb rearrange_dataset_v2 --replace --data-path data/ --no-prune | |
| ls -la data/scene_datasets/habitat-test-scenes/ | |
| - name: Run sim benchmark | |
| run: |- | |
| export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH | |
| conda init | |
| source ~/.bashrc | |
| conda activate habitat | |
| cd habitat-sim | |
| python examples/example.py --scene data/scene_datasets/habitat-test-scenes/van-gogh-room.glb --silent --test_fps_regression $FPS_THRESHOLD | |
| - name: Run sim tests | |
| run: |- | |
| #give cmake ownership to the runner for installation | |
| sudo chown runner -R /opt/cmake312/ | |
| #activate conda env | |
| export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH | |
| conda init | |
| source ~/.bashrc | |
| conda activate habitat | |
| cd habitat-sim | |
| export PYTHONPATH=$(pwd):$PYTHONPATH | |
| #set pytest to show partial progress | |
| export PYTHONUNBUFFERED=1 | |
| # run tests with code coverage | |
| CORRADE_TEST_COLOR=ON GTEST_COLOR=yes ./build.sh --headless \ | |
| --bullet \ | |
| --with-cuda \ | |
| --build-datatool \ | |
| --run-tests \ | |
| --no-lto \ | |
| --cmake-args='-DCMAKE_CXX_FLAGS="--coverage"' | |
| PYTHONPATH=src_python pytest -n 4 --durations=10 --cov-report=xml --cov=./ | |
| #run the marked pytest-benchmark tests and print the results | |
| PYTHONPATH=src_python pytest -m sim_benchmarks | |
| #re-build without bullet and cuda and run physics tests again | |
| #TODO: instead of reinstall, do this with configuration | |
| ./build.sh --headless --cmake-args='-DCMAKE_CXX_FLAGS="--coverage"' | |
| PYTHONPATH=src_python pytest -n 4 --durations=10 --cov-report=xml --cov=./ --cov-append tests/test_physics.py tests/test_sensors.py | |
| - name: upload test coverage | |
| run: |- | |
| export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH | |
| conda init | |
| source ~/.bashrc | |
| conda activate habitat | |
| cd habitat-sim | |
| curl -Os https://uploader.codecov.io/latest/linux/codecov | |
| chmod +x codecov | |
| #Uploading test coverage for Python code | |
| ./codecov -f coverage.xml -cF Python | |
| #Uploading test coverage for C++ code | |
| lcov --directory . --capture --output-file coverage.info | |
| # Replaces -1 linecount with zero to prevent lcov from crashing: | |
| # https://github.com/psycofdj/coverxygen/issues/6 | |
| sed -i -e 's/,-1$/,0/g' coverage.info | |
| #lcov --remove coverage.info "*/deps/*" --output-file coverage.info > /dev/null | |
| #lcov --remove coverage.info "*/test/*" --output-file coverage.info > /dev/null | |
| #lcov --remove coverage.info "*/tests/*" --output-file coverage.info > /dev/null | |
| ./codecov -f coverage.info -cF CPP | |
| - name: install habitat-sim with audio and run audio_agent script | |
| run: |- | |
| #give cmake ownership to the runner for installation | |
| sudo chown runner -R /opt/cmake312/ | |
| export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH | |
| conda init | |
| source ~/.bashrc | |
| conda activate habitat | |
| cd habitat-sim | |
| pip install -r requirements.txt --progress-bar off | |
| pip install imageio imageio-ffmpeg | |
| git submodule update --init --recursive --jobs 8 | |
| python -u setup.py install --build-type "Release" --lto --headless --audio | |
| python examples/tutorials/audio_agent.py | |
| #NOTE: use the below to debug with ssh: simply move this "job" just before the crashing job to intercept the workflow | |
| #- name: Debugging with tmate | |
| # uses: mxschmitt/action-tmate@v3.18 | |
| build_conda_binaries: | |
| runs-on: 4-core-ubuntu-gpu-t4 | |
| env: | |
| AIHABITAT_CONDA_USERNAME: aihabitat | |
| AIHABITAT_CONDA_PWD: ${{ secrets.AIHABITAT_CONDA_PWD }} | |
| AIHABITAT_NIGHTLY_CONDA_USERNAME: aihabitat-nightly | |
| AIHABITAT_NIGHTLY_CONDA_PWD: ${{ secrets.AIHABITAT_NIGHTLY_CONDA_PWD }} | |
| defaults: | |
| run: | |
| shell: bash -el {0} | |
| steps: | |
| - uses: actions/checkout@v4.1.1 | |
| with: | |
| path: "./habitat-sim" | |
| - name: Set CI_TEST variable based on "main merge" vs. "pull_request" workflow | |
| run: |- | |
| echo "NIGHTLY=''" >> $GITHUB_ENV | |
| echo "CI_TEST='true'" >> $GITHUB_ENV | |
| if [[ ${{ github.event_name }} == 'push' ]]; then | |
| if [[ ${GITHUB_REF} == refs/heads/* ]]; then | |
| echo "trigger: a push to main branch" | |
| elif [[ ${GITHUB_REF} == refs/tags/* ]]; then | |
| echo "trigger: a version tag push" | |
| echo "CI_TEST='false'" >> $GITHUB_ENV | |
| echo "CONDA_USERNAME=$AIHABITAT_CONDA_USERNAME" >> $GITHUB_ENV | |
| echo "CONDA_PWD=$AIHABITAT_CONDA_PWD" >> $GITHUB_ENV | |
| fi | |
| elif [[ ${{ github.event_name }} == 'schedule' ]]; then | |
| echo "trigger: a nightly build" | |
| echo "CI_TEST='false'" >> $GITHUB_ENV | |
| echo "NIGHTLY='--nightly'" >> $GITHUB_ENV | |
| echo "CONDA_USERNAME=$AIHABITAT_NIGHTLY_CONDA_USERNAME" >> $GITHUB_ENV | |
| echo "CONDA_PWD=$AIHABITAT_NIGHTLY_CONDA_PWD" >> $GITHUB_ENV | |
| elif [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
| echo "trigger: a pull request" | |
| fi | |
| - name: Build conda Linux packages | |
| run: |- | |
| cd habitat-sim/conda-build | |
| echo "Current event name: $GITHUB_EVENT_NAME" | |
| echo ${{ env.CI_TEST }} | |
| echo ${{ env.NIGHTLY }} | |
| #pivot on the build type | |
| if [[ ${{ env.CI_TEST }} ]]; then | |
| echo "This is a CI test." | |
| else | |
| echo "This is a full conda build with deployment." | |
| # Install anaconda to work with packages repo | |
| curl -O https://repo.anaconda.com/archive/Anaconda3-2020.11-Linux-x86_64.sh | |
| bash Anaconda3-2020.11-Linux-x86_64.sh -b -p $HOME/anaconda | |
| export PATH=$HOME/anaconda/bin:$PATH | |
| # Delete old nightly builds | |
| #NOTE: The ${{ env.NIGHTLY }} == "--nightly" variable is necessary to trigger this | |
| #python common/delete_old_night_packages.py --username ${{ env.CONDA_USERNAME }} --password ${{ env.CONDA_PWD }} ${{ env.NIGHTLY }} | |
| fi | |
| #install Docker | |
| curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - | |
| sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | |
| sudo apt-get update | |
| apt-cache policy docker-ce | |
| sudo apt-get install -y docker-ce | |
| #build the container | |
| docker build -t hsim_condabuild_dcontainer -f Dockerfile . | |
| if [[ ${{ env.CI_TEST }} ]]; then | |
| docker run --ipc=host --rm -v $(pwd)/../../:/remote hsim_condabuild_dcontainer /bin/bash -c "source ~/.bashrc && conda activate py39 && cd /remote/habitat-sim/ && git config --global --add safe.directory '*' && cd /remote/habitat-sim/conda-build && python matrix_builder.py --ci_test" | |
| else | |
| docker run --ipc=host --rm -v $(pwd)/../../:/remote --env ${{ env.CONDA_PWD }} hsim_condabuild_dcontainer /bin/bash -c "source ~/.bashrc && conda activate py39 && cd /remote/habitat-sim/ && git config --global --add safe.directory '*' && cd /remote/habitat-sim/conda-build && yes | anaconda login --username ${{ env.CONDA_USERNAME }} --password ${{ env.CONDA_PWD }} --hostname "aihabitat-conda-ci-builder-linux" && python matrix_builder.py --conda_upload ${{ env.NIGHTLY }}" | |
| fi | |
| build_conda_osx: | |
| runs-on: macos-14 | |
| env: | |
| AIHABITAT_CONDA_USERNAME: aihabitat | |
| AIHABITAT_CONDA_PWD: ${{ secrets.AIHABITAT_CONDA_PWD }} | |
| AIHABITAT_NIGHTLY_CONDA_USERNAME: aihabitat-nightly | |
| AIHABITAT_NIGHTLY_CONDA_PWD: ${{ secrets.AIHABITAT_NIGHTLY_CONDA_PWD }} | |
| defaults: | |
| run: | |
| shell: bash -el {0} | |
| steps: | |
| - uses: actions/checkout@v4.1.1 | |
| with: | |
| path: "./habitat-sim" | |
| - name: Setup python | |
| uses: actions/setup-python@v5.0.0 | |
| with: | |
| python-version: '3.9.20' | |
| - name: Conda Install OSX | |
| run: |- | |
| curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-py310_24.1.2-0-MacOSX-arm64.sh | |
| chmod +x ~/miniconda.sh | |
| ~/miniconda.sh -b -p $HOME/miniconda | |
| rm ~/miniconda.sh | |
| export PATH=$HOME/miniconda/bin:$PATH | |
| conda init "$(basename "${SHELL}")" | |
| - name: Set CI_TEST variable based on "main merge" vs. "pull_request" workflow | |
| run: |- | |
| echo "NIGHTLY=''" >> $GITHUB_ENV | |
| echo "CI_TEST='true'" >> $GITHUB_ENV | |
| if [[ ${{ github.event_name }} == 'push' ]]; then | |
| if [[ ${GITHUB_REF} == refs/heads/* ]]; then | |
| echo "trigger: a push to main branch" | |
| elif [[ ${GITHUB_REF} == refs/tags/* ]]; then | |
| echo "trigger: a version tag push" | |
| echo "CI_TEST='false'" >> $GITHUB_ENV | |
| echo "CONDA_USERNAME=$AIHABITAT_CONDA_USERNAME" >> $GITHUB_ENV | |
| echo "CONDA_PWD=$AIHABITAT_CONDA_PWD" >> $GITHUB_ENV | |
| fi | |
| elif [[ ${{ github.event_name }} == 'schedule' ]]; then | |
| echo "trigger: a nightly build" | |
| echo "CI_TEST='false'" >> $GITHUB_ENV | |
| echo "NIGHTLY='--nightly'" >> $GITHUB_ENV | |
| echo "CONDA_USERNAME=$AIHABITAT_NIGHTLY_CONDA_USERNAME" >> $GITHUB_ENV | |
| echo "CONDA_PWD=$AIHABITAT_NIGHTLY_CONDA_PWD" >> $GITHUB_ENV | |
| elif [[ ${{ github.event_name }} == 'pull_request' ]]; then | |
| echo "trigger: a pull request" | |
| fi | |
| - name: Build conda OSX packages | |
| run: |- | |
| export PATH=$HOME/miniconda/bin:$PATH | |
| source ~/.bash_profile | |
| conda activate | |
| conda install -y anaconda-client ccache cmake git ninja conda-build pip | |
| echo $(which -a python) | |
| pip install gitpython | |
| cd habitat-sim | |
| git submodule update --init --recursive --jobs 8 | |
| pip install . | |
| pip uninstall -y habitat-sim | |
| rm -rf build | |
| (yes || true) | anaconda login --username ${{ env.CONDA_USERNAME }} --password ${{ env.CONDA_PWD }} --hostname "aihabitat-conda-ci-builder-macos" | |
| #pivot on the build type | |
| if [[ ${{ env.CI_TEST }} ]]; then | |
| echo "This is a CI test." | |
| cd conda-build | |
| export PYTHONIOENCODING="utf-8" | |
| python matrix_builder.py --ci_test | |
| else | |
| conda config --set anaconda_upload yes | |
| cd conda-build | |
| export PYTHONIOENCODING="utf-8" | |
| python matrix_builder.py ${{ env.NIGHTLY }} | |
| fi |