From b54232349026965230f7e909b12a7a98c44f2056 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Fri, 5 Mar 2021 17:50:21 +0100 Subject: [PATCH] CI: fix examples - patch download MNIST (#6357) * patch download * CI * isort * extra remove runif --- .github/workflows/ci_test-full.yml | 7 +- azure-pipelines.yml | 16 +- pl_examples/__init__.py | 16 ++ pl_examples/basic_examples/autoencoder.py | 4 +- .../backbone_image_classifier.py | 4 +- .../basic_examples/dali_image_classifier.py | 12 +- .../basic_examples/mnist_datamodule.py | 4 +- .../generative_adversarial_net.py | 12 +- .../training_type/training_type_plugin.py | 2 +- tests/plugins/test_sharded_plugin.py | 1 - tests/utilities/test_parsing.py | 209 +++++++++++++++--- 11 files changed, 236 insertions(+), 51 deletions(-) diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index f08c277b71064..45540ca12fa5f 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -143,10 +143,9 @@ jobs: # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003 coverage run --source pytorch_lightning -m pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml - # todo: put this back just when TorchVision can download datasets - #- name: Examples - # run: | - # python -m pytest pl_examples -v --durations=10 + - name: Examples + run: | + python -m pytest pl_examples -v --durations=10 - name: Upload pytest test results uses: actions/upload-artifact@v2 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6d67afc31f2e4..48db9ede12400 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -100,10 +100,12 @@ jobs: python -m pytest benchmarks -v --maxfail=2 --durations=0 displayName: 'Testing: benchmarks' - # todo: put this back just when TorchVision can download datasets - #- bash: | - # python -m pytest pl_examples -v --maxfail=2 --durations=0 - # python setup.py install --user --quiet - # bash pl_examples/run_ddp-example.sh - # pip uninstall -y pytorch-lightning - # displayName: 'Examples' + - bash: | + python -m pytest pl_examples -v --maxfail=2 --durations=0 + python setup.py install --user --quiet + bash pl_examples/run_ddp-example.sh + cd pl_examples/basic_examples + bash submit_ddp_job.sh + bash submit_ddp2_job.sh + pip uninstall -y pytorch-lightning + displayName: 'Examples' diff --git a/pl_examples/__init__.py b/pl_examples/__init__.py index 6ad0a4dfc0624..ffd60f9ed71af 100644 --- a/pl_examples/__init__.py +++ b/pl_examples/__init__.py @@ -1,14 +1,30 @@ import os +from urllib.error import HTTPError + +from six.moves import urllib from pytorch_lightning.utilities import _module_available +# TorchVision hotfix https://github.com/pytorch/vision/issues/1938 +opener = urllib.request.build_opener() +opener.addheaders = [('User-agent', 'Mozilla/5.0')] +urllib.request.install_opener(opener) + _EXAMPLES_ROOT = os.path.dirname(__file__) _PACKAGE_ROOT = os.path.dirname(_EXAMPLES_ROOT) _DATASETS_PATH = os.path.join(_PACKAGE_ROOT, 'Datasets') _TORCHVISION_AVAILABLE = _module_available("torchvision") +_TORCHVISION_MNIST_AVAILABLE = True _DALI_AVAILABLE = _module_available("nvidia.dali") +if _TORCHVISION_AVAILABLE: + try: + from torchvision.datasets.mnist import MNIST + MNIST(_DATASETS_PATH, download=True) + except HTTPError: + _TORCHVISION_MNIST_AVAILABLE = False + LIGHTNING_LOGO = """ #### ########### diff --git a/pl_examples/basic_examples/autoencoder.py b/pl_examples/basic_examples/autoencoder.py index c60c4faec4acd..b3188a21b7f04 100644 --- a/pl_examples/basic_examples/autoencoder.py +++ b/pl_examples/basic_examples/autoencoder.py @@ -20,9 +20,9 @@ from torch.utils.data import DataLoader, random_split import pytorch_lightning as pl -from pl_examples import _DATASETS_PATH, _TORCHVISION_AVAILABLE, cli_lightning_logo +from pl_examples import _DATASETS_PATH, _TORCHVISION_AVAILABLE, _TORCHVISION_MNIST_AVAILABLE, cli_lightning_logo -if _TORCHVISION_AVAILABLE: +if _TORCHVISION_AVAILABLE and _TORCHVISION_MNIST_AVAILABLE: from torchvision import transforms from torchvision.datasets.mnist import MNIST else: diff --git a/pl_examples/basic_examples/backbone_image_classifier.py b/pl_examples/basic_examples/backbone_image_classifier.py index ad50da18ff3fd..01a5dca0de3c7 100644 --- a/pl_examples/basic_examples/backbone_image_classifier.py +++ b/pl_examples/basic_examples/backbone_image_classifier.py @@ -19,9 +19,9 @@ from torch.utils.data import DataLoader, random_split import pytorch_lightning as pl -from pl_examples import _DATASETS_PATH, _TORCHVISION_AVAILABLE, cli_lightning_logo +from pl_examples import _DATASETS_PATH, _TORCHVISION_AVAILABLE, _TORCHVISION_MNIST_AVAILABLE, cli_lightning_logo -if _TORCHVISION_AVAILABLE: +if _TORCHVISION_AVAILABLE and _TORCHVISION_MNIST_AVAILABLE: from torchvision import transforms from torchvision.datasets.mnist import MNIST else: diff --git a/pl_examples/basic_examples/dali_image_classifier.py b/pl_examples/basic_examples/dali_image_classifier.py index d6e64d2b3de14..b4bf1407a9b26 100644 --- a/pl_examples/basic_examples/dali_image_classifier.py +++ b/pl_examples/basic_examples/dali_image_classifier.py @@ -23,9 +23,15 @@ from torch.utils.data import random_split import pytorch_lightning as pl -from pl_examples import _DALI_AVAILABLE, _DATASETS_PATH, _TORCHVISION_AVAILABLE, cli_lightning_logo - -if _TORCHVISION_AVAILABLE: +from pl_examples import ( + _DALI_AVAILABLE, + _DATASETS_PATH, + _TORCHVISION_AVAILABLE, + _TORCHVISION_MNIST_AVAILABLE, + cli_lightning_logo, +) + +if _TORCHVISION_AVAILABLE and _TORCHVISION_MNIST_AVAILABLE: from torchvision import transforms from torchvision.datasets.mnist import MNIST else: diff --git a/pl_examples/basic_examples/mnist_datamodule.py b/pl_examples/basic_examples/mnist_datamodule.py index 46acc5a3a2a14..a50f67cdab301 100644 --- a/pl_examples/basic_examples/mnist_datamodule.py +++ b/pl_examples/basic_examples/mnist_datamodule.py @@ -17,10 +17,10 @@ from torch.utils.data import DataLoader, random_split -from pl_examples import _DATASETS_PATH, _TORCHVISION_AVAILABLE +from pl_examples import _DATASETS_PATH, _TORCHVISION_AVAILABLE, _TORCHVISION_MNIST_AVAILABLE from pytorch_lightning import LightningDataModule -if _TORCHVISION_AVAILABLE: +if _TORCHVISION_AVAILABLE and _TORCHVISION_MNIST_AVAILABLE: from torchvision import transforms as transform_lib from torchvision.datasets import MNIST else: diff --git a/pl_examples/domain_templates/generative_adversarial_net.py b/pl_examples/domain_templates/generative_adversarial_net.py index 2aa2a9f73db8b..285fba8b93f1b 100644 --- a/pl_examples/domain_templates/generative_adversarial_net.py +++ b/pl_examples/domain_templates/generative_adversarial_net.py @@ -26,15 +26,19 @@ import torch import torch.nn as nn import torch.nn.functional as F # noqa -import torchvision -import torchvision.transforms as transforms from torch.utils.data import DataLoader -from torchvision.datasets import MNIST -from pl_examples import cli_lightning_logo +from pl_examples import _TORCHVISION_AVAILABLE, _TORCHVISION_MNIST_AVAILABLE, cli_lightning_logo from pytorch_lightning.core import LightningDataModule, LightningModule from pytorch_lightning.trainer import Trainer +if _TORCHVISION_AVAILABLE and _TORCHVISION_MNIST_AVAILABLE: + import torchvision + import torchvision.transforms as transforms + from torchvision.datasets import MNIST +else: + from tests.helpers.datasets import MNIST + class Generator(nn.Module): """ diff --git a/pytorch_lightning/plugins/training_type/training_type_plugin.py b/pytorch_lightning/plugins/training_type/training_type_plugin.py index d7c3b4d4d77e1..4c84ec8cbd5b4 100644 --- a/pytorch_lightning/plugins/training_type/training_type_plugin.py +++ b/pytorch_lightning/plugins/training_type/training_type_plugin.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from abc import ABC, abstractmethod -from typing import Any, Callable, Iterable, Optional, TYPE_CHECKING, Union +from typing import Any, Callable, Dict, Iterable, Optional, TYPE_CHECKING, Union import torch from torch.nn import Module diff --git a/tests/plugins/test_sharded_plugin.py b/tests/plugins/test_sharded_plugin.py index bca5079f82f82..d431779dddb4e 100644 --- a/tests/plugins/test_sharded_plugin.py +++ b/tests/plugins/test_sharded_plugin.py @@ -31,7 +31,6 @@ def test_ddp_sharded_precision_16_clip_gradients(mock_oss_clip_grad_norm, clip_v mock_oss_clip_grad_norm.assert_not_called() -@RunIf(fairscale=True) @pytest.mark.parametrize(["accelerator"], [("ddp_sharded", ), ("ddp_sharded_spawn", )]) @pytest.mark.skipif(not _FAIRSCALE_AVAILABLE, reason="Fairscale is not available") def test_sharded_ddp_choice(tmpdir, accelerator): diff --git a/tests/utilities/test_parsing.py b/tests/utilities/test_parsing.py index 42edb8e48f336..7087d183906a1 100644 --- a/tests/utilities/test_parsing.py +++ b/tests/utilities/test_parsing.py @@ -1,22 +1,27 @@ -# Copyright The PyTorch Lightning team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +import inspect + import pytest -from pytorch_lightning.utilities.parsing import lightning_getattr, lightning_hasattr, lightning_setattr +from pytorch_lightning.utilities.parsing import ( + AttributeDict, + clean_namespace, + collect_init_args, + flatten_dict, + get_init_args, + is_picklable, + lightning_getattr, + lightning_hasattr, + lightning_setattr, + parse_class_init_keys, + str_to_bool, + str_to_bool_or_str, +) + +unpicklable_function = lambda: None -def _get_test_cases(): +@pytest.fixture(scope="module") +def model_cases(): class TestHparamsNamespace: learning_rate = 1 @@ -74,9 +79,9 @@ class TestModel7: # test for datamodule w/ hparams w/ attribute (should use dat return model1, model2, model3, model4, model5, model6, model7 -def test_lightning_hasattr(tmpdir): +def test_lightning_hasattr(tmpdir, model_cases): """Test that the lightning_hasattr works in all cases""" - model1, model2, model3, model4, model5, model6, model7 = models = _get_test_cases() + model1, model2, model3, model4, model5, model6, model7 = models = model_cases assert lightning_hasattr(model1, 'learning_rate'), \ 'lightning_hasattr failed to find namespace variable' assert lightning_hasattr(model2, 'learning_rate'), \ @@ -96,9 +101,9 @@ def test_lightning_hasattr(tmpdir): assert not lightning_hasattr(m, "this_attr_not_exist") -def test_lightning_getattr(tmpdir): +def test_lightning_getattr(tmpdir, model_cases): """Test that the lightning_getattr works in all cases""" - models = _get_test_cases() + models = model_cases for i, m in enumerate(models[:3]): value = lightning_getattr(m, 'learning_rate') assert value == i, 'attribute not correctly extracted' @@ -113,15 +118,15 @@ def test_lightning_getattr(tmpdir): for m in models: with pytest.raises( - AttributeError, - match="is neither stored in the model namespace nor the `hparams` namespace/dict, nor the datamodule." + AttributeError, + match="is neither stored in the model namespace nor the `hparams` namespace/dict, nor the datamodule." ): lightning_getattr(m, "this_attr_not_exist") -def test_lightning_setattr(tmpdir): +def test_lightning_setattr(tmpdir, model_cases): """Test that the lightning_setattr works in all cases""" - models = _get_test_cases() + models = model_cases for m in models[:3]: lightning_setattr(m, 'learning_rate', 10) assert lightning_getattr(m, 'learning_rate') == 10, \ @@ -140,7 +145,161 @@ def test_lightning_setattr(tmpdir): for m in models: with pytest.raises( - AttributeError, - match="is neither stored in the model namespace nor the `hparams` namespace/dict, nor the datamodule." + AttributeError, + match="is neither stored in the model namespace nor the `hparams` namespace/dict, nor the datamodule." ): lightning_setattr(m, "this_attr_not_exist", None) + + +def test_str_to_bool_or_str(tmpdir): + true_cases = ['y', 'yes', 't', 'true', 'on', '1'] + false_cases = ['n', 'no', 'f', 'false', 'off', '0'] + other_cases = ['yyeess', 'noooo', 'lightning'] + + for case in true_cases: + assert str_to_bool_or_str(case) is True + + for case in false_cases: + assert str_to_bool_or_str(case) is False + + for case in other_cases: + assert str_to_bool_or_str(case) == case + + +def test_str_to_bool(tmpdir): + true_cases = ['y', 'yes', 't', 'true', 'on', '1'] + false_cases = ['n', 'no', 'f', 'false', 'off', '0'] + other_cases = ['yyeess', 'noooo', 'lightning'] + + for case in true_cases: + assert str_to_bool(case) is True + + for case in false_cases: + assert str_to_bool(case) is False + + for case in other_cases: + with pytest.raises(ValueError): + str_to_bool(case) + + +def test_is_picklable(tmpdir): + # See the full list of picklable types at + # https://docs.python.org/3/library/pickle.html#pickle-picklable + class UnpicklableClass: + # Only classes defined at the top level of a module are picklable. + pass + + true_cases = [None, True, 123, "str", (123, "str"), max] + false_cases = [unpicklable_function, UnpicklableClass] + + for case in true_cases: + assert is_picklable(case) is True + + for case in false_cases: + assert is_picklable(case) is False + + +def test_clean_namespace(tmpdir): + # See the full list of picklable types at + # https://docs.python.org/3/library/pickle.html#pickle-picklable + class UnpicklableClass: + # Only classes defined at the top level of a module are picklable. + pass + + test_case = { + "1": None, + "2": True, + "3": 123, + "4": unpicklable_function, + "5": UnpicklableClass, + } + + clean_namespace(test_case) + + assert test_case == {"1": None, "2": True, "3": 123} + + +def test_parse_class_init_keys(tmpdir): + + class Class: + + def __init__(self, hparams, *my_args, anykw=42, **my_kwargs): + pass + + assert parse_class_init_keys(Class) == ("self", "my_args", "my_kwargs") + + +def test_get_init_args(tmpdir): + + class AutomaticArgsModel: + + def __init__(self, anyarg, anykw=42, **kwargs): + super().__init__() + + self.get_init_args_wrapper() + + def get_init_args_wrapper(self): + frame = inspect.currentframe().f_back + self.result = get_init_args(frame) + + my_class = AutomaticArgsModel("test", anykw=32, otherkw=123) + assert my_class.result == {"anyarg": "test", "anykw": 32, "otherkw": 123} + + my_class.get_init_args_wrapper() + assert my_class.result == {} + + +def test_collect_init_args(): + + class AutomaticArgsParent: + + def __init__(self, anyarg, anykw=42, **kwargs): + super().__init__() + self.get_init_args_wrapper() + + def get_init_args_wrapper(self): + frame = inspect.currentframe() + self.result = collect_init_args(frame, []) + + class AutomaticArgsChild(AutomaticArgsParent): + + def __init__(self, anyarg, childarg, anykw=42, childkw=42, **kwargs): + super().__init__(anyarg, anykw=anykw, **kwargs) + + my_class = AutomaticArgsChild("test1", "test2", anykw=32, childkw=22, otherkw=123) + assert my_class.result[0] == {"anyarg": "test1", "anykw": 32, "otherkw": 123} + assert my_class.result[1] == {"anyarg": "test1", "childarg": "test2", "anykw": 32, "childkw": 22, "otherkw": 123} + + +def test_attribute_dict(tmpdir): + # Test initialization + inputs = { + 'key1': 1, + 'key2': 'abc', + } + ad = AttributeDict(inputs) + for key, value in inputs.items(): + assert getattr(ad, key) == value + + # Test adding new items + ad = AttributeDict() + ad.update({'key1': 1}) + assert ad.key1 == 1 + + # Test updating existing items + ad = AttributeDict({'key1': 1}) + ad.key1 = 123 + assert ad.key1 == 123 + + +def test_flatten_dict(tmpdir): + d = {'1': 1, '_': {'2': 2, '_': {'3': 3, '4': 4}}} + + expected = { + '1': 1, + '2': 2, + '3': 3, + '4': 4, + } + + assert flatten_dict(d) == expected