CI: fix examples - patch download MNIST (#6357)

* patch download

* CI

* isort

* extra

remove runif

.github/workflows/ci_test-full.yml

@@ -143,10 +143,9 @@ jobs:
  # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
  coverage run --source pytorch_lightning -m pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml
 
- # todo: put this back just when TorchVision can download datasets
- #- name: Examples
- # run: |
- # python -m pytest pl_examples -v --durations=10
+ - name: Examples
+ run: |
+ python -m pytest pl_examples -v --durations=10
 
  - name: Upload pytest test results
  uses: actions/upload-artifact@v2

azure-pipelines.yml

@@ -100,10 +100,12 @@ jobs:
  python -m pytest benchmarks -v --maxfail=2 --durations=0
  displayName: 'Testing: benchmarks'
 
- # todo: put this back just when TorchVision can download datasets
- #- bash: |
- # python -m pytest pl_examples -v --maxfail=2 --durations=0
- # python setup.py install --user --quiet
- # bash pl_examples/run_ddp-example.sh
- # pip uninstall -y pytorch-lightning
- # displayName: 'Examples'
+ - bash: |
+ python -m pytest pl_examples -v --maxfail=2 --durations=0
+ python setup.py install --user --quiet
+ bash pl_examples/run_ddp-example.sh
+ cd pl_examples/basic_examples
+ bash submit_ddp_job.sh
+ bash submit_ddp2_job.sh
+ pip uninstall -y pytorch-lightning
+ displayName: 'Examples'

pl_examples/__init__.py

@@ -1,14 +1,30 @@
 import os
+from urllib.error import HTTPError
+
+from six.moves import urllib
 
 from pytorch_lightning.utilities import _module_available
 
+# TorchVision hotfix https://github.com/pytorch/vision/issues/1938
+opener = urllib.request.build_opener()
+opener.addheaders = [('User-agent', 'Mozilla/5.0')]
+urllib.request.install_opener(opener)
+
 _EXAMPLES_ROOT = os.path.dirname(__file__)
 _PACKAGE_ROOT = os.path.dirname(_EXAMPLES_ROOT)
 _DATASETS_PATH = os.path.join(_PACKAGE_ROOT, 'Datasets')
 
 _TORCHVISION_AVAILABLE = _module_available("torchvision")
+_TORCHVISION_MNIST_AVAILABLE = True
 _DALI_AVAILABLE = _module_available("nvidia.dali")
 
+if _TORCHVISION_AVAILABLE:
+ try:
+ from torchvision.datasets.mnist import MNIST
+ MNIST(_DATASETS_PATH, download=True)
+ except HTTPError:
+ _TORCHVISION_MNIST_AVAILABLE = False
+
 LIGHTNING_LOGO = """
  ####
  ###########

pl_examples/basic_examples/autoencoder.py

@@ -20,9 +20,9 @@
 from torch.utils.data import DataLoader, random_split
 
 import pytorch_lightning as pl
-from pl_examples import _DATASETS_PATH, _TORCHVISION_AVAILABLE, cli_lightning_logo
+from pl_examples import _DATASETS_PATH, _TORCHVISION_AVAILABLE, _TORCHVISION_MNIST_AVAILABLE, cli_lightning_logo
 
-if _TORCHVISION_AVAILABLE:
+if _TORCHVISION_AVAILABLE and _TORCHVISION_MNIST_AVAILABLE:
  from torchvision import transforms
  from torchvision.datasets.mnist import MNIST
 else:

pl_examples/basic_examples/backbone_image_classifier.py

@@ -19,9 +19,9 @@
 from torch.utils.data import DataLoader, random_split
 
 import pytorch_lightning as pl
-from pl_examples import _DATASETS_PATH, _TORCHVISION_AVAILABLE, cli_lightning_logo
+from pl_examples import _DATASETS_PATH, _TORCHVISION_AVAILABLE, _TORCHVISION_MNIST_AVAILABLE, cli_lightning_logo
 
-if _TORCHVISION_AVAILABLE:
+if _TORCHVISION_AVAILABLE and _TORCHVISION_MNIST_AVAILABLE:
  from torchvision import transforms
  from torchvision.datasets.mnist import MNIST
 else:

pl_examples/basic_examples/dali_image_classifier.py

@@ -23,9 +23,15 @@
 from torch.utils.data import random_split
 
 import pytorch_lightning as pl
-from pl_examples import _DALI_AVAILABLE, _DATASETS_PATH, _TORCHVISION_AVAILABLE, cli_lightning_logo
-
-if _TORCHVISION_AVAILABLE:
+from pl_examples import (
+ _DALI_AVAILABLE,
+ _DATASETS_PATH,
+ _TORCHVISION_AVAILABLE,
+ _TORCHVISION_MNIST_AVAILABLE,
+ cli_lightning_logo,
+)
+
+if _TORCHVISION_AVAILABLE and _TORCHVISION_MNIST_AVAILABLE:
  from torchvision import transforms
  from torchvision.datasets.mnist import MNIST
 else:

pl_examples/basic_examples/mnist_datamodule.py

@@ -17,10 +17,10 @@
 
 from torch.utils.data import DataLoader, random_split
 
-from pl_examples import _DATASETS_PATH, _TORCHVISION_AVAILABLE
+from pl_examples import _DATASETS_PATH, _TORCHVISION_AVAILABLE, _TORCHVISION_MNIST_AVAILABLE
 from pytorch_lightning import LightningDataModule
 
-if _TORCHVISION_AVAILABLE:
+if _TORCHVISION_AVAILABLE and _TORCHVISION_MNIST_AVAILABLE:
  from torchvision import transforms as transform_lib
  from torchvision.datasets import MNIST
 else:

pl_examples/domain_templates/generative_adversarial_net.py

@@ -26,15 +26,19 @@
 import torch
 import torch.nn as nn
 import torch.nn.functional as F # noqa
-import torchvision
-import torchvision.transforms as transforms
 from torch.utils.data import DataLoader
-from torchvision.datasets import MNIST
 
-from pl_examples import cli_lightning_logo
+from pl_examples import _TORCHVISION_AVAILABLE, _TORCHVISION_MNIST_AVAILABLE, cli_lightning_logo
 from pytorch_lightning.core import LightningDataModule, LightningModule
 from pytorch_lightning.trainer import Trainer
 
+if _TORCHVISION_AVAILABLE and _TORCHVISION_MNIST_AVAILABLE:
+ import torchvision
+ import torchvision.transforms as transforms
+ from torchvision.datasets import MNIST
+else:
+ from tests.helpers.datasets import MNIST
+
 
 class Generator(nn.Module):
  """

pytorch_lightning/plugins/training_type/training_type_plugin.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from abc import ABC, abstractmethod
-from typing import Any, Callable, Iterable, Optional, TYPE_CHECKING, Union
+from typing import Any, Callable, Dict, Iterable, Optional, TYPE_CHECKING, Union
 
 import torch
 from torch.nn import Module

tests/plugins/test_sharded_plugin.py

@@ -31,7 +31,6 @@ def test_ddp_sharded_precision_16_clip_gradients(mock_oss_clip_grad_norm, clip_v
  mock_oss_clip_grad_norm.assert_not_called()
 
 
-@RunIf(fairscale=True)
 @pytest.mark.parametrize(["accelerator"], [("ddp_sharded", ), ("ddp_sharded_spawn", )])
 @pytest.mark.skipif(not _FAIRSCALE_AVAILABLE, reason="Fairscale is not available")
 def test_sharded_ddp_choice(tmpdir, accelerator):