require: adjust versions (#6363)

* adjust versions

* release

* manifest

* pep8

* CI

* fix

* build

.github/workflows/ci_test-conda.yml

@@ -27,24 +27,20 @@ jobs:
  run: |
  conda info
  conda list
+ # adjust versions according installed Torch version
+ python ./requirements/adjust_versions.py requirements/extra.txt
+ python ./requirements/adjust_versions.py requirements/examples.txt
  pip install --requirement requirements/devel.txt --upgrade-strategy only-if-needed
  pip list
 
  - name: Pull checkpoints from S3
- # todo: consider adding coma caching, but ATM all models have less then 100KB
  run: |
  # enter legacy and update checkpoints from S3
  cd legacy
  curl https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip --output checkpoints.zip
  unzip -o checkpoints.zip
  ls -l checkpoints/
 
- # todo: require proper fix in docker image
- - name: Hotfix dependency
- run: |
- pip install torchtext==0.6.0 -U
- shell: bash
-
  - name: Tests
  run: |
  # NOTE: run coverage on tests does not propagare faler status for Win, https://github.com/nedbat/coveragepy/issues/1003

.github/workflows/ci_test-full.yml

@@ -104,20 +104,17 @@ jobs:
  HOROVOD_WITHOUT_MXNET: 1
  HOROVOD_WITHOUT_TENSORFLOW: 1
  run: |
- # python -m pip install --upgrade --user pip
- pip install --requirement requirements.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet --upgrade
- pip install --requirement ./requirements/devel.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --quiet --upgrade
  python --version
  pip --version
+ # python -m pip install --upgrade --user pip
+ pip install --requirement requirements.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
+ # adjust versions according installed Torch version
+ python ./requirements/adjust_versions.py requirements/extra.txt
+ python ./requirements/adjust_versions.py requirements/examples.txt
+ pip install --requirement ./requirements/devel.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
  pip list
  shell: bash
 
- # todo: require proper fix in docker image
- - name: Hotfix dependency
- run: |
- pip install torchtext==0.6.0 -U
- shell: bash
-
  - name: Reinstall Horovod if necessary
  if: runner.os != 'windows'
  env:

.github/workflows/events-nightly.yml

@@ -102,8 +102,6 @@ jobs:
  id: extend
 
  - name: Publish CUDA to Docker Hub
- # ToDo: extend also building for Nightly from pip
- if: matrix.pytorch_version < 1.8
  # publish master/release
  uses: docker/build-push-action@v2
  with:

MANIFEST.in

@@ -46,7 +46,7 @@ recursive-include docs/source/_static/images/general/ pl_overview* tf_* tutorial
 
 # Include the Requirements
 recursive-include requirements *.txt
-recursive-exclude requirements *.sh
+recursive-exclude requirements *.sh *.py
 include requirements.txt
 include pyproject.toml
 

azure-pipelines.yml

@@ -71,11 +71,6 @@ jobs:
  python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu >= 2, f'GPU: {mgpu}'"
  displayName: 'Env details'
 
- # todo: require proper fix in docker image
- - bash: |
- pip install torchtext==0.7 -U
- displayName: 'HotFix'
-
  - bash: |
  wget https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip -P legacy/
  unzip -o legacy/checkpoints.zip -d legacy/

dockers/base-conda/Dockerfile

@@ -98,10 +98,12 @@ ENV \
 
 COPY ./requirements/extra.txt requirements-extra.txt
 COPY ./requirements/test.txt requirements-test.txt
+COPY ./requirements/adjust_versions.py requirements_adjust_versions.py
 
 RUN \
  pip list | grep torch && \
  python -c "import torch; print(torch.__version__)" && \
+ python requirements_adjust_versions.py requirements-extra.txt && \
  # Install remaining requirements
  pip install -r requirements-extra.txt --no-cache-dir && \
  pip install -r requirements-test.txt --no-cache-dir && \

dockers/base-cuda/Dockerfile

@@ -94,12 +94,14 @@ RUN \
  # eventualy use pre-release
  #pip install "torch==${PYTORCH_VERSION}.*" --pre && \
  # set particular PyTorch version
- python -c "import re ; fname = 'requirements.txt' ; req = re.sub(r'torch[>=]+[\d\.]+', 'torch==${PYTORCH_VERSION}.*', open(fname).read()) ; open(fname, 'w').write(req)" && \
+ python ./requirements/adjust_versions.py requirements.txt ${PYTORCH_VERSION} && \
+ python ./requirements/adjust_versions.py requirements/extra.txt ${PYTORCH_VERSION} && \
+ python ./requirements/adjust_versions.py requirements/examples.txt ${PYTORCH_VERSION} && \
  # Install all requirements
  # todo: find a way how to install nightly PT version
  # --pre --extra-index-url https://download.pytorch.org/whl/nightly/cu${cuda_ver[0]}${cuda_ver[1]}/torch_nightly.html
  pip install -r requirements/devel.txt --no-cache-dir && \
- rm -rf requirements*
+ rm -rf requirements.* requirements/
 
 RUN \
  # install DALI, needed for examples
@@ -113,7 +115,7 @@ RUN \
 
 RUN \
  # install DeepSpeed from source.
- # todo: swap to pypi release once DeepSpeed releases a new version.
+ # todo: swap to pypi release once DeepSpeed releases a new version >= 0.3.10
  pip install deepspeed@git+https://github.com/microsoft/DeepSpeed@ec8b1cb
 
 RUN \

dockers/base-xla/Dockerfile

@@ -104,6 +104,7 @@ RUN \
  python -c "fname = 'requirements/extra.txt' ; lines = [line for line in open(fname).readlines() if 'fairscale' not in line] ; open(fname, 'w').writelines(lines)" && \
  # drop TorchVision as it was installed with XLA
  python -c "fname = 'requirements/examples.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('torchvision')] ; open(fname, 'w').writelines(lines)" && \
+ python ./requirements/adjust_versions.py ./requirements/extra.txt && \
  pip install --requirement ./requirements/devel.txt --no-cache-dir && \
  cd .. && \
  rm -rf pytorch-lightning && \

dockers/release/Dockerfile

@@ -27,14 +27,15 @@ COPY ./ ./pytorch-lightning/
 RUN \
  # Disable cache
  #conda install "pip>20.1" && \
- #pip config set global.cache-dir false && \
- if [ -z $LIGHTNING_VERSION ] ; then \
- pip install ./pytorch-lightning --no-cache-dir ; \
+ if [ ! -z "$LIGHTNING_VERSION" ] ; then \
  rm -rf pytorch-lightning ; \
- else \
- rm -rf pytorch-lightning ; \
- pip install https://github.com/PyTorchLightning/pytorch-lightning/archive/${LIGHTNING_VERSION}.zip --no-cache-dir ; \
- fi
+ wget https://github.com/PyTorchLightning/pytorch-lightning/archive/${LIGHTNING_VERSION}.zip --progress=bar:force:noscroll ; \
+ unzip ${LIGHTNING_VERSION}.zip ; \
+ mv pytorch-lightning-*/ pytorch-lightning ; \
+ rm *.zip ; \
+ fi && \
+ pip install ./pytorch-lightning["extra"] --no-cache-dir && \
+ rm -rf pytorch-lightning
 
 RUN python --version && \
  pip --version && \

requirements/adjust_versions.py

@@ -0,0 +1,49 @@
+import os
+import re
+import sys
+from typing import Any, Dict
+
+VERSIONS_LUT: Dict[str, Dict[str, Any]] = {
+ "1.4.0": dict(torchvision="0.5.0", torchtext="0.5"),
+ "1.5.0": dict(torchvision="0.6.0", torchtext="0.6"),
+ "1.5.1": dict(torchvision="0.6.1", torchtext="0.6"),
+ "1.6.0": dict(torchvision="0.7.0", torchtext="0.7"),
+ "1.7.0": dict(torchvision="0.8.1", torchtext="0.8"),
+ "1.7.1": dict(torchvision="0.8.2", torchtext="0.8.1"),
+ "1.8.0": dict(torchvision="0.9.0", torchtext="0.9"),
+}
+
+
+def find_latest(ver: str, versions_all: list) -> str:
+ # drop all except semantic version
+ ver = re.search(r'([\.\d]+)', ver).groups()[0]
+ # find candidates, by starting version pattern
+ options = [v for v in versions_all if v.startswith(ver)]
+ assert options, f"missing {ver} among {versions_all}"
+ # take the last one...
+ return sorted(options)[-1]
+
+
+def main(path_req: str, torch_version: str = None) -> None:
+ with open(path_req, "r") as fp:
+ req = fp.read()
+
+ if not torch_version:
+ import torch
+ torch_version = torch.__version__
+ assert torch_version, f"invalid/missing Torch: {torch_version}"
+
+ torch_version = find_latest(torch_version, list(VERSIONS_LUT.keys()))
+ dep_versions = VERSIONS_LUT[torch_version]
+ dep_versions["torch"] = torch_version
+ for lib in dep_versions:
+ version = dep_versions[lib]
+ replace = f"{lib}=={version}\n"
+ req = re.sub(rf"{lib}[>=]*[\d\.]*{os.linesep}", replace, req)
+
+ with open(path_req, "w") as fp:
+ fp.write(req)
+
+
+if __name__ == "__main__":
+ main(*sys.argv[1:])