Drone: use nightly build cuda docker images

.drone.yml

@@ -20,44 +20,21 @@ name: torch-GPU
 
 steps:
 - name: testing
-  image: pytorchlightning/pytorch_lightning:cuda-extras-py3.7-torch1.5
+  image: pytorchlightning/pytorch_lightning:base-cuda-py3.7-torch1.5
 
   environment:
-    SLURM_LOCALID: 0
     CODECOV_TOKEN:
       from_secret: codecov_token
     MKL_THREADING_LAYER: GNU
-    HOROVOD_GPU_OPERATIONS: NCCL
-    HOROVOD_WITH_PYTORCH: 1
-    HOROVOD_WITHOUT_TENSORFLOW: 1
-    HOROVOD_WITHOUT_MXNET: 1
-    HOROVOD_WITH_GLOO: 1
-    HOROVOD_WITHOUT_MPI: 1
-
-  #volumes:
-  #  # Mount pip cache from host
-  #  - name: pip_cache
-  #    path: /opt/conda/lib/python3.7/site-packages
 
   commands:
-    # todo: remove unsets as in correct image Horovod shall be set
-    - unset HOROVOD_GPU_ALLREDUCE
-    - unset HOROVOD_GPU_BROADCAST
-    - export PATH="$PATH:/root/.local/bin"
     - python --version
-    - pip install pip -U
     - pip --version
     - nvidia-smi
-    #- bash ./requirements/install_AMP.sh
-    - apt-get update && apt-get install -y cmake
-    - pip uninstall -y horovod  # todo: this shall not be needed
-    - pip install -r ./requirements/devel.txt --user -q --upgrade-strategy only-if-needed  --no-cache-dir
-    #- pip install -r ./requirements/docs.txt --user -q
-    - pip install -r ./requirements/examples.txt --user -q --upgrade-strategy only-if-needed
+    - pip install -r ./requirements/devel.txt --upgrade-strategy only-if-needed -v --no-cache-dir
     - pip list
-    - python -c "import torch ; print(' & '.join([torch.cuda.get_device_name(i) for i in range(torch.cuda.device_count())]) if torch.cuda.is_available() else 'only CPU')"
     - coverage run --source pytorch_lightning -m pytest pytorch_lightning tests -v --color=yes --durations=25 # --flake8
-    - python -m py.test benchmarks pl_examples -v --color=yes --maxfail=2 --durations=0 # --flake8
+    - python -m pytest benchmarks pl_examples -v --color=yes --maxfail=2 --durations=0 # --flake8
     #- cd docs; make doctest; make coverage
     - coverage report
     # see: https://docs.codecov.io/docs/merging-reports
@@ -73,8 +50,3 @@ trigger:
     include:
     - push
     - pull_request
-
-#volumes:
-#  - name: pip_cache
-#    host:
-#      path: /tmp/cache/drone/pip

.github/workflows/ci_dockers.yml

@@ -9,7 +9,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
     branches: [master]
 
 jobs:
-  build-Conda:
+  build-PL:
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
@@ -21,18 +21,16 @@ jobs:
         uses: actions/checkout@v2
 
       # https://github.com/docker/setup-buildx-action
-      # to use cache-from and cache-to argument of buildx command
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Build Conda Docker
+      # Set up Docker Buildx - to use cache-from and cache-to argument of buildx command
+      - uses: docker/setup-buildx-action@v1
+      - name: Build PL Docker
         # publish master
         uses: docker/build-push-action@v2
         with:
           build-args: |
             PYTHON_VERSION=${{ matrix.python_version }}
             PYTORCH_VERSION=${{ matrix.pytorch_version }}
-          file: dockers/conda/Dockerfile
+          file: dockers/release/Dockerfile
           push: false
         timeout-minutes: 50
 
@@ -48,10 +46,8 @@ jobs:
         uses: actions/checkout@v2
 
       # https://github.com/docker/setup-buildx-action
-      # to use cache-from and cache-to argument of buildx command
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
+      # Set up Docker Buildx - to use cache-from and cache-to argument of buildx command
+      - uses: docker/setup-buildx-action@v1
       - name: Build XLA Docker
         # publish master
         uses: docker/build-push-action@v2
@@ -70,33 +66,75 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          #- python_version: 3.7
-          #  pytorch_version: 1.8  # todo
-          #  pytorch_channel: pytorch-nightly
-          - python_version: 3.8
+          #- python_version: 3.8
+          #  pytorch_version: 1.7  # todo
+          - python_version: 3.7
             pytorch_version: 1.6
-            pytorch_channel: pytorch
           - python_version: 3.6
-            pytorch_version: 1.5
-            pytorch_channel: pytorch
+            pytorch_version: 1.3
     steps:
       - name: Checkout
         uses: actions/checkout@v2
 
-      # https://github.com/docker/setup-buildx-action
-      # to use cache-from and cache-to argument of buildx command
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+      # for PT 1.3 and 1.4 we need to use CUDA 10.1
+      - run: |
+          cuda=$(python -c "print(10.2 if float(${{matrix.pytorch_version}}) > 1.4 else 10.1)" 2>&1)
+          echo "::set-output name=CUDA::$cuda"
+        id: extend
 
+      # https://github.com/docker/setup-buildx-action
+      # Set up Docker Buildx - to use cache-from and cache-to argument of buildx command
+      - uses: docker/setup-buildx-action@v1
       - name: Build CUDA Docker
         # publish master
         uses: docker/build-push-action@v2
         with:
           build-args: |
             PYTHON_VERSION=${{ matrix.python_version }}
             PYTORCH_VERSION=${{ matrix.pytorch_version }}
-            PYTORCH_CHANNEL=${{ matrix.pytorch_channel }}
+            CUDA_VERSION=${{ steps.extend.outputs.CUDA }}
           cache-from: pytorchlightning/pytorch_lightning:base-cuda-cache-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}
           file: dockers/base-cuda/Dockerfile
           push: false
         timeout-minutes: 50
+
+  build-conda:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - python_version: 3.8
+            pytorch_version: 1.6
+          - python_version: 3.6
+            pytorch_version: 1.4
+          #- python_version: 3.7
+          #  pytorch_version: 1.8  # todo
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      # for PT 1.3 and 1.4 we need to use CUDA 10.1
+      - run: |
+          cuda=$(python -c "print(10.2 if float(${{matrix.pytorch_version}}) > 1.4 else 10.1)" 2>&1)
+          echo "::set-output name=CUDA::$cuda"
+          channel=$(python -c "print('pytorch-nightly' if float(${{matrix.pytorch_version}}) > 1.7 else 'pytorch')" 2>&1)
+          echo "::set-output name=CHANNEL::$channel"
+        id: extend
+
+      # https://github.com/docker/setup-buildx-action
+      # Set up Docker Buildx - to use cache-from and cache-to argument of buildx command
+      - uses: docker/setup-buildx-action@v1
+      - name: Build CUDA Docker
+        # publish master
+        uses: docker/build-push-action@v2
+        with:
+          build-args: |
+            PYTHON_VERSION=${{ matrix.python_version }}
+            PYTORCH_VERSION=${{ matrix.pytorch_version }}
+            PYTORCH_CHANNEL=${{ steps.extend.outputs.CHANNEL }}
+            CUDA_VERSION=${{ steps.extend.outputs.CUDA }}
+          cache-from: pytorchlightning/pytorch_lightning:base-conda-cache-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}
+          file: dockers/base-conda/Dockerfile
+          push: false
+        timeout-minutes: 50

.github/workflows/ci_test-conda.yml

@@ -9,14 +9,14 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
 
 jobs:
   conda:
-    runs-on: ${{ matrix.os }}
-    container: pytorchlightning/pytorch_lightning:base-cuda-py${{ matrix.python-version }}-torch${{ matrix.pytorch-version }}
+    runs-on: ubuntu-20.04
+    container: pytorchlightning/pytorch_lightning:base-conda-py${{ matrix.python-version }}-torch${{ matrix.pytorch-version }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04]
+        # os: [ubuntu-20.04]
         python-version: [3.7]
-        pytorch-version: [1.3, 1.4, 1.5, 1.6, 1.7]
+        pytorch-version: [1.3, 1.4, 1.5, 1.6]  # , 1.7 # todo
 
     # Timeout: https://stackoverflow.com/a/59076067/4521646
     timeout-minutes: 35

.github/workflows/docker-builds.yml

@@ -8,7 +8,7 @@ on:
     types: [created]
 
 jobs:
-  build-Conda:
+  build-PL:
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
@@ -36,7 +36,7 @@ jobs:
           repository: pytorchlightning/pytorch_lightning
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-          dockerfile: dockers/conda/Dockerfile
+          dockerfile: dockers/release/Dockerfile
           build_args: PYTHON_VERSION=${{ matrix.python_version }},PYTORCH_VERSION=${{ matrix.pytorch_version }},LIGHTNING_VERSION=${{ env.RELEASE_VERSION }}
           tags: "${{ env.RELEASE_VERSION }}-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }},latest-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}"
         timeout-minutes: 55

.github/workflows/nightly.yml

@@ -8,6 +8,7 @@ on:
 
 # based on https://github.com/pypa/gh-action-pypi-publish
 jobs:
+
   pypi-release:
     runs-on: ubuntu-20.04
 
@@ -47,10 +48,8 @@ jobs:
         uses: actions/checkout@v2
 
       # https://github.com/docker/setup-buildx-action
-      # to use cache-from and cache-to argument of buildx command
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
+      # Set up Docker Buildx - to use cache-from and cache-to argument of buildx command
+      - uses: docker/setup-buildx-action@v1
       - name: Login to DockerHub
         uses: docker/login-action@v1
         with:
@@ -78,45 +77,40 @@ jobs:
       matrix:
         python_version: [3.6, 3.7, 3.8]
         pytorch_version: [1.3, 1.4, 1.5, 1.6]  # todo: , 1.7
-        pytorch_channel: ["pytorch", "pytorch-nightly"]
-        # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#example-including-new-combinations
         exclude:
-          - pytorch_version: 1.7
-            pytorch_channel: pytorch
-          - pytorch_version: 1.3
-            pytorch_channel: pytorch-nightly
-          - pytorch_version: 1.4
-            pytorch_channel: pytorch-nightly
-          - pytorch_version: 1.5
-            pytorch_channel: pytorch-nightly
-          - pytorch_version: 1.6
-            pytorch_channel: pytorch-nightly
-          - pytorch_version: 1.3
-            pytorch_channel: pytorch
-            python_version: 3.8
+          # excludes PT 1.3 as it is missing on pypi
+          - python_version: 3.8
+            pytorch_version: 1.3
+
     steps:
       - name: Checkout
         uses: actions/checkout@v2
 
       # https://github.com/docker/setup-buildx-action
-      # to use cache-from and cache-to argument of buildx command
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-
+      # Set up Docker Buildx - to use cache-from and cache-to argument of buildx command
+      - uses: docker/setup-buildx-action@v1
       - name: Login to DockerHub
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
+      # for PT 1.3 and 1.4 we need to use CUDA 10.1
+      - run: |
+          cuda=$(python -c "print(10.2 if float(${{matrix.pytorch_version}}) > 1.4 else 10.1)" 2>&1)
+          echo "::set-output name=CUDA::$cuda"
+          channel=$(python -c "print('pytorch-nightly' if float(${{matrix.pytorch_version}}) > 1.7 else 'pytorch')" 2>&1)
+          echo "::set-output name=CHANNEL::$channel"
+        id: extend
+
       - name: Publish CUDA to Docker Hub
         # publish master
         uses: docker/build-push-action@v2
         with:
           build-args: |
             PYTHON_VERSION=${{ matrix.python_version }}
             PYTORCH_VERSION=${{ matrix.pytorch_version }}
-            PYTORCH_CHANNEL=${{ matrix.pytorch_channel }}
+            CUDA_VERSION=${{ steps.extend.outputs.CUDA }}
           cache-from: pytorchlightning/pytorch_lightning:base-cuda-cache-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}
           cache-to: pytorchlightning/pytorch_lightning:base-cuda-cache-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}
           file: dockers/base-cuda/Dockerfile
@@ -131,7 +125,11 @@ jobs:
           build-args: |
             PYTHON_VERSION=${{ matrix.python_version }}
             PYTORCH_VERSION=${{ matrix.pytorch_version }}
-          file: dockers/conda/Dockerfile
+            PYTORCH_CHANNEL=${{ steps.extend.outputs.CHANNEL }}
+            CUDA_VERSION=${{ steps.extend.outputs.CUDA }}
+          cache-from: pytorchlightning/pytorch_lightning:base-conda-cache-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}
+          cache-to: pytorchlightning/pytorch_lightning:base-conda-cache-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}
+          file: dockers/base-conda/Dockerfile
           push: true
-          tags: pytorchlightning/pytorch_lightning:nightly-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}
+          tags: pytorchlightning/pytorch_lightning:base-conda-py${{ matrix.python_version }}-torch${{ matrix.pytorch_version }}
         timeout-minutes: 55

README.md

@@ -91,8 +91,8 @@ Lightning can automatically export to ONNX or TorchScript for those cases.
 
 | System / PyTorch ver. | 1.3 (min. req.)* | 1.4 | 1.5 | 1.6 (latest) | 1.7 (nightly) |
 | :---: | :---: | :---: | :---: | :---: | :---: |
-| Conda py3.7 [linux] | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg?event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg?event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg?event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg?event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg?event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) |
-| Linux py3.7 [GPUs**] | - | - |[![Build Status](http://104.154.220.231/api/badges/PyTorchLightning/pytorch-lightning/status.svg)](http://104.154.220.231/PyTorchLightning/pytorch-lightning) | - | - |
+| Conda py3.7 [linux] | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | - |
+| Linux py3.7 [GPUs**] | - | - | [![Build Status](http://104.154.220.231/api/badges/PyTorchLightning/pytorch-lightning/status.svg)](http://104.154.220.231/PyTorchLightning/pytorch-lightning) | - | - |
 | Linux py3.7 [TPUs***] | - | - | - | [![TPU tests](https://github.com/PyTorchLightning/pytorch-lightning/workflows/TPU%20tests/badge.svg)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22TPU+tests%22+branch%3Amaster) | - |
 | Linux py3.6 / py3.7 / py3.8 | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - |
 | OSX py3.6 / py3.7 | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - |

dockers/README.md

@@ -1,4 +1,6 @@
-## Builds
+# Docker images
+
+## Builds images form attached Dockerfiles
 
 You can build it on your own, note it takes lots of time, be prepared.
 
@@ -31,4 +33,23 @@ and if you do not need it anymore, just clean it:
 ```bash
 docker image list
 docker image rm pytorch-lightning:latest
-```
+```
+
+### Run docker image with GPUs
+
+To run docker image with access to you GPUs you need to install
+```bash
+# Add the package repositories
+distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
+curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
+curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
+
+sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit
+sudo systemctl restart docker
+```
+
+and later run the docker image with `--gpus all` so for example
+
+```
+docker run --rm -it --gpus all pytorchlightning/pytorch_lightning:base-cuda-py3.7-torch1.6
+```