Merge pull request #339 from Project-MONAI/dev

merge master

.github/pull_request_template.md

@@ -12,6 +12,6 @@ A few sentences describing the changes proposed in this pull request.
 - [ ] Breaking change (fix or new feature that would cause existing functionality to change).
 - [ ] New tests added to cover the changes.
 - [ ] Integration tests passed locally by running `./runtests.sh -f -u --net --coverage`.
-- [ ] Quick tests passed locally by running `./runtests.sh --quick --unittests`.
+- [ ] Quick tests passed locally by running `./runtests.sh --quick --unittests  --disttests`.
 - [ ] In-line docstrings updated.
 - [ ] Documentation updated, tested `make html` command in the `docs/` folder.

.github/workflows/cron-mmar.yml

@@ -0,0 +1,42 @@
+name: cron-mmar
+
+on:
+  schedule:
+    - cron: "0 2 * * *"  # at 02:00 UTC
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+concurrency:
+  # automatically cancel the previously triggered workflows when there's a newer version
+  group: mmar-tests-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  cron-load:
+    if: github.repository == 'Project-MONAI/MONAI'
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: cache weekly timestamp
+      id: pip-cache
+      run: echo "::set-output name=datew::$(date '+%Y-%V')"
+    - name: cache for pip
+      uses: actions/cache@v2
+      id: cache
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ steps.pip-cache.outputs.datew }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip wheel
+        python -m pip install -r requirements-dev.txt
+    - name: Loading MMARs
+      run: |
+        # clean up temporary files
+        $(pwd)/runtests.sh --build --clean
+        # run tests
+        python -m tests.ngc_mmar_loading

.github/workflows/cron.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: [self-hosted, linux, x64, common]
     strategy:
       matrix:
-        pytorch-version: [1.5.1, 1.6.0, 1.7.1, 1.8.1, latest]
+        pytorch-version: [1.6.0, 1.7.1, 1.8.1, 1.9.1, latest]
     steps:
     - uses: actions/checkout@v2
     - name: Install the dependencies
@@ -25,14 +25,14 @@ jobs:
         python -m pip uninstall -y torch torchvision
         if [ ${{ matrix.pytorch-version }} == "latest" ]; then
           python -m pip install torch torchvision
-        elif [ ${{ matrix.pytorch-version }} == "1.5.1" ]; then
-          python -m pip install torch==1.5.1 torchvision==0.6.1
         elif [ ${{ matrix.pytorch-version }} == "1.6.0" ]; then
           python -m pip install torch==1.6.0 torchvision==0.7.0
         elif [ ${{ matrix.pytorch-version }} == "1.7.1" ]; then
           python -m pip install torch==1.7.1 torchvision==0.8.2
         elif [ ${{ matrix.pytorch-version }} == "1.8.1" ]; then
           python -m pip install torch==1.8.1 torchvision==0.9.1
+        elif [ ${{ matrix.pytorch-version }} == "1.9.1" ]; then
+          python -m pip install torch==1.9.1 torchvision==0.10.1
         fi
         python -m pip install -r requirements-dev.txt
         python -m pip list
@@ -48,8 +48,8 @@ jobs:
         python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
         python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))'
-        BUILD_MONAI=1 ./runtests.sh --coverage --unittests  # unit tests with coverage report
-        BUILD_MONAI=1 ./runtests.sh --coverage --net  # integration tests with coverage report
+        BUILD_MONAI=1 ./runtests.sh --build --coverage --unittests --disttests  # unit tests with coverage report
+        BUILD_MONAI=1 ./runtests.sh --build --coverage --net  # integration tests with coverage report
         coverage xml
         if pgrep python; then pkill python; fi
     - name: Upload coverage
@@ -62,7 +62,7 @@ jobs:
     if: github.repository == 'Project-MONAI/MONAI'
     strategy:
       matrix:
-        container: ["pytorch:21.02", "pytorch:21.08"]  # 21.02 for backward comp.
+        container: ["pytorch:21.02", "pytorch:21.12"]  # 21.02 for backward comp.
     container:
       image: nvcr.io/nvidia/${{ matrix.container }}-py3  # testing with the latest pytorch base image
       options: "--gpus all"
@@ -91,8 +91,8 @@ jobs:
         python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
         python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))'
-        BUILD_MONAI=1 ./runtests.sh --coverage --unittests  # unit tests with coverage report
-        BUILD_MONAI=1 ./runtests.sh --coverage --net  # integration tests with coverage report
+        BUILD_MONAI=1 ./runtests.sh --build --coverage --unittests --disttests  # unit tests with coverage report
+        BUILD_MONAI=1 ./runtests.sh --build --coverage --net  # integration tests with coverage report
         coverage xml
         if pgrep python; then pkill python; fi
     - name: Upload coverage
@@ -106,7 +106,7 @@ jobs:
     if: github.repository == 'Project-MONAI/MONAI'
     strategy:
       matrix:
-        container: ["pytorch:21.02", "pytorch:21.08"]  # 21.02 for backward comp.
+        container: ["pytorch:21.02", "pytorch:21.12"]  # 21.02 for backward comp.
     container:
       image: nvcr.io/nvidia/${{ matrix.container }}-py3  # testing with the latest pytorch base image
       options: "--gpus all"
@@ -173,7 +173,7 @@ jobs:
   cron-docker:
     if: github.repository == 'Project-MONAI/MONAI'
     container:
-      image: localhost:5000/local_monai:dockerhub # use currently latest, locally available dockerhub image
+      image: docker://projectmonai/monai:latest  # this might be slow and has the pull count limitations
       options: "--gpus all"
     runs-on: [self-hosted, linux, x64, common]
     steps:
@@ -190,8 +190,8 @@ jobs:
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
         python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
         ngc --version
-        BUILD_MONAI=1 ./runtests.sh --coverage --pytype --unittests  # unit tests with pytype checks, coverage report
-        BUILD_MONAI=1 ./runtests.sh --coverage --net  # integration tests with coverage report
+        BUILD_MONAI=1 ./runtests.sh --build --coverage --pytype --unittests --disttests  # unit tests with pytype checks, coverage report
+        BUILD_MONAI=1 ./runtests.sh --build --coverage --net  # integration tests with coverage report
         coverage xml
         if pgrep python; then pkill python; fi
     - name: Upload coverage
@@ -204,7 +204,7 @@ jobs:
     if: github.repository == 'Project-MONAI/MONAI'
     needs: cron-gpu  # so that monai itself is verified first
     container:
-      image: nvcr.io/nvidia/pytorch:21.08-py3  # testing with the latest pytorch base image
+      image: nvcr.io/nvidia/pytorch:21.09-py3  # testing with the latest pytorch base image
       options: "--gpus all --ipc=host"
     runs-on: [self-hosted, linux, x64, common]
     steps:
@@ -215,7 +215,7 @@ jobs:
         which python
         python -m pip install --upgrade pip wheel
         python -m pip install -r requirements-dev.txt
-        BUILD_MONAI=0 python setup.py develop  # install monai
+        BUILD_MONAI=1 python setup.py develop  # install monai
         nvidia-smi
         export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
         echo $CUDA_VISIBLE_DEVICES
@@ -234,5 +234,7 @@ jobs:
         trap 'if pgrep python; then pkill python; fi;' ERR
         python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
         cd /opt/tutorials
+        python -c 'import monai; monai.config.print_debug_info()'
         $(pwd)/runner.sh
+        python -c 'import monai; monai.config.print_debug_info()'
         if pgrep python; then pkill python; fi

.github/workflows/docker.yml

@@ -13,20 +13,21 @@ on:
   workflow_dispatch:
 
 jobs:
-  versioning:
+  versioning_dev:
     # compute versioning file from python setup.py
     # upload as artifact
-    # (also used in release.yml)
     if: github.repository == 'Project-MONAI/MONAI'
-    container:
-      image: localhost:5000/local_monai:latest
-    runs-on: [self-hosted, linux, x64, build_only]
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
         # full history so that we can git describe
         with:
           ref: dev
           fetch-depth: 0
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
       - shell: bash
         run: |
           git describe
@@ -43,13 +44,11 @@ jobs:
           ls -al
           rm -rf {*,.[^.]*}
 
-  local_docker:
-    # builds two versions: local_monai:latest and local_monai:dockerhub
-    # latest: used for local tests
-    # dockerhub: release, no flake package
+  docker_build_dev:
+    # builds projectmonai/monai:latest
     if: github.repository == 'Project-MONAI/MONAI'
-    needs: versioning
-    runs-on: [self-hosted, linux, x64, build_only]
+    needs: versioning_dev
+    runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
       with:
@@ -58,67 +57,47 @@ jobs:
       uses: actions/download-artifact@v2
       with:
         name: _version.py
+    - name: Install Latest Docker
+      run: |
+        curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+        sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu  $(lsb_release -cs)  stable"
+        sudo apt-get update
+        sudo apt-get install docker-ce
     - name: docker_build
       shell: bash
       run: |
         # get tag info for versioning
         cat _version.py
         mv _version.py monai/
-        # build and run original docker image for local registry
-        docker build -t localhost:5000/local_monai:latest -f Dockerfile .
-        docker push localhost:5000/local_monai:latest
-        # build once more w/ tag "latest": remove flake package as it is not needed on hub.docker.com
+
+        # build "latest": remove flake package as it is not needed on hub.docker.com
         sed -i '/flake/d' requirements-dev.txt
         docker build -t projectmonai/monai:latest -f Dockerfile .
-        # also push as tag "dockerhub" to local registry
-        docker image tag projectmonai/monai:latest localhost:5000/local_monai:dockerhub
-        docker push localhost:5000/local_monai:dockerhub
+
         # distribute as always w/ tag "latest" to hub.docker.com
         echo "${{ secrets.DOCKER_PW }}" | docker login -u projectmonai --password-stdin
+
         docker push projectmonai/monai:latest
         docker logout
         docker image prune -f
 
-  docker_test_latest:
-    if: github.repository == 'Project-MONAI/MONAI'
-    needs: local_docker
-    container:
-      image: localhost:5000/local_monai:latest
-    runs-on: [self-hosted, linux, x64, common]
-    steps:
-    - name: Import
-      run: |
-        export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
-        echo $CUDA_VISIBLE_DEVICES
-        trap 'if pgrep python; then pkill python; fi;' ERR
-        python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
-        python -c 'import monai; monai.config.print_config()'
-        cd /opt/monai
-        ls -al
-        ngc --version
-        python -m tests.min_tests
-        if pgrep python; then pkill python; fi
-      env:
-        QUICKTEST: True
-
   docker_test_dockerhub:
     if: github.repository == 'Project-MONAI/MONAI'
-    needs: local_docker
+    needs: docker_build_dev
     container:
-      image: localhost:5000/local_monai:dockerhub
-    runs-on: [self-hosted, linux, x64, common]
+      image: docker://projectmonai/monai:latest
+      options: "--shm-size=4g --ipc=host"
+    runs-on: ubuntu-latest
     steps:
     - name: Import
       run: |
         export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
         echo $CUDA_VISIBLE_DEVICES
-        trap 'if pgrep python; then pkill python; fi;' ERR
-        python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
-        python -c 'import monai; monai.config.print_config()'
+        python -c 'import monai; monai.config.print_debug_info()'
         cd /opt/monai
         ls -al
         ngc --version
-        python -m tests.min_tests
-        if pgrep python; then pkill python; fi
+        ./runtests.sh --min
+      shell: bash
       env:
         QUICKTEST: True

.github/workflows/integration.yml

@@ -34,7 +34,7 @@ jobs:
         which python
         python -m pip install --upgrade pip wheel
         python -m pip uninstall -y torch torchvision
-        python -m pip install torch==1.9.0+cu111 torchvision==0.10.0+cu111 -f https://download.pytorch.org/whl/torch_stable.html
+        python -m pip install torch==1.10.1+cu111 torchvision==0.11.2+cu111 -f https://download.pytorch.org/whl/torch_stable.html
         python -m pip install -r requirements-dev.txt
     - name: Run integration tests
       run: |
@@ -46,8 +46,8 @@ jobs:
         python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
         python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
-        BUILD_MONAI=1 ./runtests.sh --net
-        BUILD_MONAI=1 ./runtests.sh --unittests
+        BUILD_MONAI=1 ./runtests.sh --build --net
+        BUILD_MONAI=1 ./runtests.sh --build --unittests --disttests
         if pgrep python; then pkill python; fi
       shell: bash
     - name: Add reaction

.github/workflows/pythonapp-gpu.yml

@@ -19,35 +19,37 @@ jobs:
     strategy:
       matrix:
         environment:
-          - "PT16+CUDA110"
           - "PT17+CUDA102"
-          - "PT17+CUDA110"
           - "PT18+CUDA102"
+          - "PT18+CUDA112"
           - "PT19+CUDA114"
-          - "PT19+CUDA102"
+          - "PT110+CUDA115"
+          - "PT110+CUDA102"
         include:
-          - environment: PT16+CUDA110
-            # we explicitly set pytorch to -h to avoid pip install error
-            pytorch: "-h"
-            base: "nvcr.io/nvidia/pytorch:20.07-py3"
+          # https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes
           - environment: PT17+CUDA102
             pytorch: "torch==1.7.1 torchvision==0.8.2"
             base: "nvcr.io/nvidia/cuda:10.2-devel-ubuntu18.04"
-          - environment: PT17+CUDA110
-            # we explicitly set pytorch to -h to avoid pip install error
-            pytorch: "-h"
-            base: "nvcr.io/nvidia/pytorch:20.09-py3"
           - environment: PT18+CUDA102
             pytorch: "torch==1.8.1 torchvision==0.9.1"
             base: "nvcr.io/nvidia/cuda:10.2-devel-ubuntu18.04"
+          - environment: PT18+CUDA112
+            # we explicitly set pytorch to -h to avoid pip install error
+            # 21.03: 1.9.0a0+df837d0
+            pytorch: "-h"
+            base: "nvcr.io/nvidia/pytorch:21.03-py3"
           - environment: PT19+CUDA114
             # we explicitly set pytorch to -h to avoid pip install error
-            # https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes
-            # 21.08: 1.10.0a0+3fd9dcf
+            # 21.10: 1.10.0a0+0aef44c
+            pytorch: "-h"
+            base: "nvcr.io/nvidia/pytorch:21.10-py3"
+          - environment: PT110+CUDA115
+            # we explicitly set pytorch to -h to avoid pip install error
+            # 21.12: 1.11.0a0+b6df043
             pytorch: "-h"
-            base: "nvcr.io/nvidia/pytorch:21.08-py3"
-          - environment: PT19+CUDA102
-            pytorch: "torch==1.9.0 torchvision==0.10.0"
+            base: "nvcr.io/nvidia/pytorch:21.12-py3"
+          - environment: PT110+CUDA102
+            pytorch: "torch==1.10.1 torchvision==0.11.2"
             base: "nvcr.io/nvidia/cuda:10.2-devel-ubuntu18.04"
     container:
       image: ${{ matrix.base }}
@@ -59,9 +61,9 @@ jobs:
       run: |
         if [ ${{ matrix.environment }} = "PT17+CUDA102" ] || \
           [ ${{ matrix.environment }} = "PT18+CUDA102" ] || \
-          [ ${{ matrix.environment }} = "PT19+CUDA102" ]
+          [ ${{ matrix.environment }} = "PT110+CUDA102" ]
         then
-        PYVER=3.6 PYSFX=3 DISTUTILS=python3-distutils && \
+        PYVER=3.7 PYSFX=3 DISTUTILS=python3-distutils && \
         apt-get update && apt-get install -y --no-install-recommends \
           curl \
           pkg-config \
@@ -100,6 +102,8 @@ jobs:
       run: |
         which python
         python -m pip install --upgrade pip wheel
+        # fixes preinstalled ruamel_yaml error from the docker image
+        rm -rf $(python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")/ruamel*
         python -m pip install ${{ matrix.pytorch }}
         python -m pip install -r requirements-dev.txt
         python -m pip list
@@ -120,8 +124,8 @@ jobs:
         python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))'
         python -c "import monai; monai.config.print_config()"
         # build for the current self-hosted CI Tesla V100
-        BUILD_MONAI=1 TORCH_CUDA_ARCH_LIST="7.0" ./runtests.sh --quick --unittests
-        if [ ${{ matrix.environment }} = "PT19+CUDA102" ]; then
+        BUILD_MONAI=1 TORCH_CUDA_ARCH_LIST="7.0" ./runtests.sh --build --quick --unittests --disttests
+        if [ ${{ matrix.environment }} = "PT110+CUDA102" ]; then
           # test the clang-format tool downloading once
           coverage run -m tests.clang_format_utils
         fi