From de7b3de6407e27360f53249549368cf6f4c546c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sun, 21 Mar 2021 21:56:55 +0100 Subject: [PATCH 001/331] setup methods --- .../accelerators/acceleratorV3.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 pytorch_lightning/accelerators/acceleratorV3.py diff --git a/pytorch_lightning/accelerators/acceleratorV3.py b/pytorch_lightning/accelerators/acceleratorV3.py new file mode 100644 index 0000000000000..648631f07dbed --- /dev/null +++ b/pytorch_lightning/accelerators/acceleratorV3.py @@ -0,0 +1,69 @@ +from collections import Callable +from typing import Any, Union + +import torch +import torch.nn as nn +from torch.optim import Optimizer +from torch.utils.data import DataLoader + +from pytorch_lightning.plugins import DDPPlugin, PrecisionPlugin +from pytorch_lightning.plugins.environments import LightningEnvironment + + +class AcceleratorV3: + + def __init__(self): + # hardcoded for a start + # this also needs to incorporate some of the accelerator connectors logic for argument handling + self.training_type_plugin = DDPPlugin( + parallel_devices=[torch.device("cuda", 0), torch.device("cuda", 1)], + num_nodes=1, + cluster_environment=LightningEnvironment(), + sync_batchnorm=False, + ) + self.precision_plugin = PrecisionPlugin() + + def setup(self, *objects: Union[nn.Module, Optimizer, DataLoader]): + # wrap all objects passed in and return them in the same order + wrapped_objects = [] + for obj in objects: + if isinstance(obj, nn.Module): + wrapped_objects.append(self.setup_model(obj)) + if isinstance(obj, Optimizer): + wrapped_objects.append(self.setup_optimizer(obj)) + if isinstance(obj, DataLoader): + wrapped_objects.append(self.setup_dataloader(obj)) + return wrapped_objects + + def setup_model(self, model: nn.Module): + # user can call this method independently instead of the general purpose setup method + pass + + def setup_optimizer(self, *optimizers: Optimizer): + # user can call this method independently instead of the general purpose setup method + pass + + def setup_dataloader(self, *dataloaders: DataLoader): + # user can call this method independently instead of the general purpose setup method + pass + + def sync(self, data: Any) -> Any: + pass + + def reduce_data(self, data: Any) -> Any: + pass + + def reduce_decision(self, decision: bool): + return False + + def broadcast_decision(self, decision: bool): + return False + + def data_to_device(self, data: Any): + pass + + def save_checkpoint(self, filepath): + pass + + def execute_on_rank(self, func: Callable, rank: int): + pass \ No newline at end of file From ea1e681fa6be6bbe3de52e99b51318f069496441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sun, 21 Mar 2021 22:49:31 +0100 Subject: [PATCH 002/331] setup model and dataloader --- .../accelerators/acceleratorV3.py | 28 +++- .../plugins/training_type/ddp.py | 28 +++- .../training_type/training_type_plugin.py | 8 + pytorch_lightning/trainer/data_loading.py | 142 +++++++++--------- 4 files changed, 124 insertions(+), 82 deletions(-) diff --git a/pytorch_lightning/accelerators/acceleratorV3.py b/pytorch_lightning/accelerators/acceleratorV3.py index 648631f07dbed..a137930820c80 100644 --- a/pytorch_lightning/accelerators/acceleratorV3.py +++ b/pytorch_lightning/accelerators/acceleratorV3.py @@ -3,11 +3,17 @@ import torch import torch.nn as nn +from torch import Tensor from torch.optim import Optimizer from torch.utils.data import DataLoader from pytorch_lightning.plugins import DDPPlugin, PrecisionPlugin from pytorch_lightning.plugins.environments import LightningEnvironment +from pytorch_lightning.utilities import move_data_to_device + + +class AcceleratedOptimizer(Optimizer): + class AcceleratorV3: @@ -23,6 +29,11 @@ def __init__(self): ) self.precision_plugin = PrecisionPlugin() + @property + def device(self): + # the device on the local rank + return self.training_type_plugin.root_device + def setup(self, *objects: Union[nn.Module, Optimizer, DataLoader]): # wrap all objects passed in and return them in the same order wrapped_objects = [] @@ -37,7 +48,8 @@ def setup(self, *objects: Union[nn.Module, Optimizer, DataLoader]): def setup_model(self, model: nn.Module): # user can call this method independently instead of the general purpose setup method - pass + model = self.training_type_plugin.setup_model(model) + return model def setup_optimizer(self, *optimizers: Optimizer): # user can call this method independently instead of the general purpose setup method @@ -45,7 +57,16 @@ def setup_optimizer(self, *optimizers: Optimizer): def setup_dataloader(self, *dataloaders: DataLoader): # user can call this method independently instead of the general purpose setup method - pass + return [self.training_type_plugin.setup_dataloader(dataloader) for dataloader in dataloaders] + + def backward(self, tensor: Tensor, *args, **kwargs): + # TODO: precision plugin backward + return tensor.backward(*args, **kwargs) + + def to_device(self, obj: Union[nn.Module, Tensor]) -> Union[nn.Module, Tensor]: + if isinstance(obj, nn.Module): + return obj.to(self.device) + return move_data_to_device(obj, device=self.device) def sync(self, data: Any) -> Any: pass @@ -59,9 +80,6 @@ def reduce_decision(self, decision: bool): def broadcast_decision(self, decision: bool): return False - def data_to_device(self, data: Any): - pass - def save_checkpoint(self, filepath): pass diff --git a/pytorch_lightning/plugins/training_type/ddp.py b/pytorch_lightning/plugins/training_type/ddp.py index bcadf16607b4f..48b7dbca20ff4 100644 --- a/pytorch_lightning/plugins/training_type/ddp.py +++ b/pytorch_lightning/plugins/training_type/ddp.py @@ -21,14 +21,17 @@ import numpy as np import torch import torch.distributed as torch_distrib +import torch.nn as nn from torch.nn.parallel.distributed import DistributedDataParallel from torch.optim import Optimizer +from torch.utils.data import DataLoader, DistributedSampler from pytorch_lightning.distributed import LightningDistributed from pytorch_lightning.overrides import LightningDistributedModule from pytorch_lightning.overrides.distributed import prepare_for_backward from pytorch_lightning.plugins.environments.cluster_environment import ClusterEnvironment from pytorch_lightning.plugins.training_type.parallel import ParallelPlugin +from pytorch_lightning.trainer.data_loading import replace_sampler from pytorch_lightning.utilities import _HYDRA_AVAILABLE, _TORCH_GREATER_EQUAL_1_7, rank_zero_warn from pytorch_lightning.utilities.distributed import rank_zero_only, ReduceOp, sync_ddp_if_available from pytorch_lightning.utilities.exceptions import MisconfigurationException @@ -90,6 +93,22 @@ def setup_environment(self): self.setup_distributed() + def setup_model(self, model: nn.Module) -> DistributedDataParallel: + self.setup_distributed() # setup distributed if it is not already initialized + model = DistributedDataParallel( + model, + device_ids=self.determine_ddp_device_ids(), + **self._ddp_kwargs, + ) + return model + + def setup_dataloader(self, dataloader: DataLoader) -> DataLoader: + self.setup_distributed() # setup distributed if it is not already initialized + kwargs = self.distributed_sampler_kwargs + sampler = DistributedSampler(dataloader.dataset, **kwargs) + dataloader = replace_sampler(dataloader, sampler) + return dataloader + def _call_children_scripts(self): # bookkeeping of spawned processes @@ -162,6 +181,9 @@ def _call_children_scripts(self): sleep(delay) def setup_distributed(self): + if torch.distributed.is_initialized(): + return + # TODO: check if needed seed = os.environ.get("PL_GLOBAL_SEED") if seed is not None: @@ -220,11 +242,7 @@ def pre_configure_ddp(self): def configure_ddp(self): self.pre_configure_ddp() - self._model = DistributedDataParallel( - LightningDistributedModule(self.model), - device_ids=self.determine_ddp_device_ids(), - **self._ddp_kwargs, - ) + self._model = self.setup_model(LightningDistributedModule(self.model)) def determine_ddp_device_ids(self): if self.root_device.type == "cpu": diff --git a/pytorch_lightning/plugins/training_type/training_type_plugin.py b/pytorch_lightning/plugins/training_type/training_type_plugin.py index 6a87792c7bd03..800d7e3f1038d 100644 --- a/pytorch_lightning/plugins/training_type/training_type_plugin.py +++ b/pytorch_lightning/plugins/training_type/training_type_plugin.py @@ -15,6 +15,7 @@ from typing import Any, Callable, Dict, Iterable, Optional, TYPE_CHECKING, Union import torch +import torch.nn as nn from torch.nn import Module from torch.optim import Optimizer from torch.utils.data import DataLoader @@ -48,6 +49,13 @@ def setup_environment(self) -> None: def setup(self, model: 'Module') -> None: """Called by the accelerator to finish setup.""" + def setup_dataloader(self, dataloader: DataLoader) -> DataLoader: + """Called by the accelerator. The plugin wraps and modifies the dataloader as needed.""" + return dataloader + + def setup_model(self, model: nn.Module) -> nn.Module: + return model + @property @abstractmethod def on_gpu(self) -> bool: diff --git a/pytorch_lightning/trainer/data_loading.py b/pytorch_lightning/trainer/data_loading.py index 56da7039bbca7..f1cdf40444ff4 100644 --- a/pytorch_lightning/trainer/data_loading.py +++ b/pytorch_lightning/trainer/data_loading.py @@ -18,7 +18,7 @@ from copy import deepcopy from typing import Callable, Iterable, List, Tuple, Union -from torch.utils.data import BatchSampler, DataLoader, RandomSampler, SequentialSampler +from torch.utils.data import BatchSampler, DataLoader, RandomSampler, SequentialSampler, Sampler from torch.utils.data.distributed import DistributedSampler from pytorch_lightning.accelerators import Accelerator @@ -84,7 +84,6 @@ def _worker_check(self, dataloader: DataLoader, name: str) -> None: ) def auto_add_sampler(self, dataloader: DataLoader, shuffle: bool) -> DataLoader: - # don't do anything if it's not a dataloader is_dataloader = isinstance(dataloader, DataLoader) # don't manipulate iterable datasets @@ -107,75 +106,8 @@ def auto_add_sampler(self, dataloader: DataLoader, shuffle: bool) -> DataLoader: # replace with distributed sampler sampler = self._get_distributed_sampler(dataloader, shuffle) - dataloader = self.replace_sampler(dataloader, sampler) - - return dataloader - - @staticmethod - def _resolve_batch_sampler(dl_args, dataloader, sampler): - batch_sampler = getattr(dataloader, "batch_sampler") - if batch_sampler is not None and type(batch_sampler) is not BatchSampler: - batch_sampler = type(batch_sampler)( - sampler, - batch_size=batch_sampler.batch_size, - drop_last=batch_sampler.drop_last, - ) - dl_args['batch_sampler'] = batch_sampler - dl_args['batch_size'] = 1 - dl_args['shuffle'] = False - dl_args['sampler'] = None - dl_args['drop_last'] = False - else: - dl_args['sampler'] = sampler - dl_args['shuffle'] = False - dl_args['batch_sampler'] = None - - return dl_args - - def replace_sampler(self, dataloader, sampler): - skip_keys = ('sampler', 'batch_sampler', 'dataset_kind') - skip_signature_keys = ('args', 'kwargs', 'self') - - attrs = {k: v for k, v in vars(dataloader).items() if not k.startswith("_")} - - params = set(inspect.signature(dataloader.__init__).parameters) - contains_dataset = True - - if type(dataloader) is not DataLoader: - contains_dataset = "dataset" in params - params.update(inspect.signature(DataLoader.__init__).parameters) - - dl_args = {name: attrs[name] for name in params if name in attrs and name not in skip_keys} - - dl_args = self._resolve_batch_sampler(dl_args, dataloader, sampler) - - multiprocessing_context = dataloader.multiprocessing_context - dl_args['multiprocessing_context'] = multiprocessing_context - - missing_kwargs = params.difference(skip_signature_keys).difference(dl_args) - if missing_kwargs: - """ - Example: - class CustomDataLoader(DataLoader): - def __init__(self, num_features, dataset, *args, **kwargs): - self.num_features = num_features - super().__init__(dataset, *args, **kwargs) - """ - dataloader_cls_name = dataloader.__class__.__name__ - raise MisconfigurationException( - f"Trying to inject DistributedSampler within {dataloader_cls_name} class." - "This would fail as your DataLoader doesn't expose all its __init__ parameters as attributes. " - f"Missing attributes are {missing_kwargs}. " - f"HINT: If you wrote the {dataloader_cls_name} class, add the `__init__` arguments as attributes or ", - "manually add DistributedSampler as " - f"{dataloader_cls_name}(dataset, ..., sampler=DistributedSampler(dataset, ...)).", - ) + dataloader = replace_sampler(dataloader, sampler) - if not contains_dataset: - dl_args.pop('dataset') - - dataloader = type(dataloader)(**dl_args) - dataloader.multiprocessing_context = multiprocessing_context return dataloader def _get_distributed_sampler(self, dataloader, shuffle): @@ -199,7 +131,7 @@ def reset_train_dataloader(self, model: LightningModule) -> None: 'You requested to overfit but enabled training dataloader shuffling.' ' We are turning it off for you.' ) - self.train_dataloader = self.replace_sampler( + self.train_dataloader = replace_sampler( self.train_dataloader, SequentialSampler(self.train_dataloader.dataset) ) @@ -298,7 +230,7 @@ def _reset_eval_dataloader( 'You requested to overfit but enabled val/test dataloader shuffling.' ' We are turning it off for you.' ) - dataloaders[loader_i] = self.replace_sampler(loader, SequentialSampler(loader.dataset)) + dataloaders[loader_i] = replace_sampler(loader, SequentialSampler(loader.dataset)) else: rank_zero_warn( @@ -405,3 +337,69 @@ def _flatten_dl_only(self, dataloaders): dataloaders = list(dataloaders) return dataloaders + + +def replace_sampler(dataloader: DataLoader, sampler: Sampler): + skip_keys = ('sampler', 'batch_sampler', 'dataset_kind') + skip_signature_keys = ('args', 'kwargs', 'self') + + attrs = {k: v for k, v in vars(dataloader).items() if not k.startswith("_")} + + params = set(inspect.signature(dataloader.__init__).parameters) + contains_dataset = True + + if type(dataloader) is not DataLoader: + contains_dataset = "dataset" in params + params.update(inspect.signature(DataLoader.__init__).parameters) + + dl_args = {name: attrs[name] for name in params if name in attrs and name not in skip_keys} + dl_args = _resolve_batch_sampler(dl_args, dataloader, sampler) + + multiprocessing_context = dataloader.multiprocessing_context + dl_args['multiprocessing_context'] = multiprocessing_context + + missing_kwargs = params.difference(skip_signature_keys).difference(dl_args) + if missing_kwargs: + """ + Example: + class CustomDataLoader(DataLoader): + def __init__(self, num_features, dataset, *args, **kwargs): + self.num_features = num_features + super().__init__(dataset, *args, **kwargs) + """ + dataloader_cls_name = dataloader.__class__.__name__ + raise MisconfigurationException( + f"Trying to inject DistributedSampler within {dataloader_cls_name} class." + "This would fail as your DataLoader doesn't expose all its __init__ parameters as attributes. " + f"Missing attributes are {missing_kwargs}. " + f"HINT: If you wrote the {dataloader_cls_name} class, add the `__init__` arguments as attributes or ", + "manually add DistributedSampler as " + f"{dataloader_cls_name}(dataset, ..., sampler=DistributedSampler(dataset, ...)).", + ) + + if not contains_dataset: + dl_args.pop('dataset') + + dataloader = type(dataloader)(**dl_args) + dataloader.multiprocessing_context = multiprocessing_context + return dataloader + + +def _resolve_batch_sampler(dl_args, dataloader, sampler): + batch_sampler = getattr(dataloader, "batch_sampler") + if batch_sampler is not None and type(batch_sampler) is not BatchSampler: + batch_sampler = type(batch_sampler)( + sampler, + batch_size=batch_sampler.batch_size, + drop_last=batch_sampler.drop_last, + ) + dl_args['batch_sampler'] = batch_sampler + dl_args['batch_size'] = 1 + dl_args['shuffle'] = False + dl_args['sampler'] = None + dl_args['drop_last'] = False + else: + dl_args['sampler'] = sampler + dl_args['shuffle'] = False + dl_args['batch_sampler'] = None + return dl_args From 0cf5fb37eaebfc0ee71d433b4527dbb7da470a3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sun, 21 Mar 2021 23:10:13 +0100 Subject: [PATCH 003/331] add optimizer wrapping --- pytorch_lightning/accelerators/acceleratorV3.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pytorch_lightning/accelerators/acceleratorV3.py b/pytorch_lightning/accelerators/acceleratorV3.py index a137930820c80..9fa20a0891639 100644 --- a/pytorch_lightning/accelerators/acceleratorV3.py +++ b/pytorch_lightning/accelerators/acceleratorV3.py @@ -1,5 +1,5 @@ from collections import Callable -from typing import Any, Union +from typing import Any, Union, Optional import torch import torch.nn as nn @@ -14,6 +14,12 @@ class AcceleratedOptimizer(Optimizer): + def __init__(self, optimizer: Optimizer): + super().__init__(params=optimizer.param_groups, default={}) # TODO: why is it called default and not defaults? + self.optimizer = optimizer + + def step(self, closure: Optional[Callable[[], float]]=...) -> Optional[float]: + return self.optimizer.step(closure) class AcceleratorV3: @@ -46,14 +52,14 @@ def setup(self, *objects: Union[nn.Module, Optimizer, DataLoader]): wrapped_objects.append(self.setup_dataloader(obj)) return wrapped_objects - def setup_model(self, model: nn.Module): + def setup_model(self, *models: nn.Module): # user can call this method independently instead of the general purpose setup method - model = self.training_type_plugin.setup_model(model) - return model + return [self.training_type_plugin.setup_model(model) for model in models] def setup_optimizer(self, *optimizers: Optimizer): # user can call this method independently instead of the general purpose setup method - pass + # TODO: let plugin setup optimizer too? + return [AcceleratedOptimizer(optimizer) for optimizer in optimizers] def setup_dataloader(self, *dataloaders: DataLoader): # user can call this method independently instead of the general purpose setup method From 1dfca227e6a4b56d187514030847a7eea0fc2654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sun, 21 Mar 2021 23:31:37 +0100 Subject: [PATCH 004/331] add basic example --- pl_examples/domain_templates/v3_example.py | 227 +++++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 pl_examples/domain_templates/v3_example.py diff --git a/pl_examples/domain_templates/v3_example.py b/pl_examples/domain_templates/v3_example.py new file mode 100644 index 0000000000000..46bc58d143db3 --- /dev/null +++ b/pl_examples/domain_templates/v3_example.py @@ -0,0 +1,227 @@ +from __future__ import print_function +import argparse +import os +import random +import torch +import torch.nn as nn +import torch.nn.parallel +import torch.optim as optim +import torch.utils.data +import torchvision.datasets as dset +import torchvision.transforms as transforms +import torchvision.utils as vutils + +parser = argparse.ArgumentParser() +parser.add_argument( + "--workers", type=int, help="number of data loading workers", default=0 +) +parser.add_argument("--batchSize", type=int, default=64, help="input batch size") +parser.add_argument( + "--imageSize", + type=int, + default=64, + help="the height / width of the input image to network", +) +parser.add_argument("--nz", type=int, default=100, help="size of the latent z vector") +parser.add_argument("--ngf", type=int, default=64) +parser.add_argument("--ndf", type=int, default=64) +parser.add_argument( + "--niter", type=int, default=25, help="number of epochs to train for" +) +parser.add_argument( + "--lr", type=float, default=0.0002, help="learning rate, default=0.0002" +) +parser.add_argument( + "--beta1", type=float, default=0.5, help="beta1 for adam. default=0.5" +) +parser.add_argument("--ngpu", type=int, default=1, help="number of GPUs to use") +parser.add_argument("--netG", default="", help="path to netG (to continue training)") +parser.add_argument("--netD", default="", help="path to netD (to continue training)") +parser.add_argument( + "--outf", default="./lightning_logs", help="folder to output images and model checkpoints" +) + +opt = parser.parse_args() +os.makedirs(opt.outf, exist_ok=True) + +nc = 1 +device = torch.device("cpu") +ngpu = int(opt.ngpu) +nz = int(opt.nz) +ngf = int(opt.ngf) +ndf = int(opt.ndf) + + +# custom weights initialization called on netG and netD +def weights_init(m): + classname = m.__class__.__name__ + if classname.find("Conv") != -1: + torch.nn.init.normal_(m.weight, 0.0, 0.02) + elif classname.find("BatchNorm") != -1: + torch.nn.init.normal_(m.weight, 1.0, 0.02) + torch.nn.init.zeros_(m.bias) + + +class Generator(nn.Module): + def __init__(self): + super(Generator, self).__init__() + self.main = nn.Sequential( + # input is Z, going into a convolution + nn.ConvTranspose2d(nz, ngf * 8, 4, 1, 0, bias=False), + nn.BatchNorm2d(ngf * 8), + nn.ReLU(True), + # state size. (ngf*8) x 4 x 4 + nn.ConvTranspose2d(ngf * 8, ngf * 4, 4, 2, 1, bias=False), + nn.BatchNorm2d(ngf * 4), + nn.ReLU(True), + # state size. (ngf*4) x 8 x 8 + nn.ConvTranspose2d(ngf * 4, ngf * 2, 4, 2, 1, bias=False), + nn.BatchNorm2d(ngf * 2), + nn.ReLU(True), + # state size. (ngf*2) x 16 x 16 + nn.ConvTranspose2d(ngf * 2, ngf, 4, 2, 1, bias=False), + nn.BatchNorm2d(ngf), + nn.ReLU(True), + # state size. (ngf) x 32 x 32 + nn.ConvTranspose2d(ngf, nc, 4, 2, 1, bias=False), + nn.Tanh() + # state size. (nc) x 64 x 64 + ) + + def forward(self, input): + return self.main(input) + +class Discriminator(nn.Module): + def __init__(self): + super(Discriminator, self).__init__() + self.main = nn.Sequential( + # input is (nc) x 64 x 64 + nn.Conv2d(nc, ndf, 4, 2, 1, bias=False), + nn.LeakyReLU(0.2, inplace=True), + # state size. (ndf) x 32 x 32 + nn.Conv2d(ndf, ndf * 2, 4, 2, 1, bias=False), + nn.BatchNorm2d(ndf * 2), + nn.LeakyReLU(0.2, inplace=True), + # state size. (ndf*2) x 16 x 16 + nn.Conv2d(ndf * 2, ndf * 4, 4, 2, 1, bias=False), + nn.BatchNorm2d(ndf * 4), + nn.LeakyReLU(0.2, inplace=True), + # state size. (ndf*4) x 8 x 8 + nn.Conv2d(ndf * 4, ndf * 8, 4, 2, 1, bias=False), + nn.BatchNorm2d(ndf * 8), + nn.LeakyReLU(0.2, inplace=True), + # state size. (ndf*8) x 4 x 4 + nn.Conv2d(ndf * 8, 1, 4, 1, 0, bias=False), + nn.Sigmoid(), + ) + + def forward(self, input): + output = self.main(input) + return output.view(-1, 1).squeeze(1) + +def main(): + random.seed(123) + torch.manual_seed(123) + + dataset = dset.MNIST( + root=".", + download=True, + transform=transforms.Compose( + [ + transforms.Resize(opt.imageSize), + transforms.ToTensor(), + transforms.Normalize((0.5,), (0.5,)), + ] + ), + ) + dataloader = torch.utils.data.DataLoader( + dataset, batch_size=opt.batchSize, shuffle=True, num_workers=opt.workers + ) + + netG = Generator().to(device) + netG.apply(weights_init) + + netD = Discriminator().to(device) + netD.apply(weights_init) + + criterion = nn.BCELoss() + + fixed_noise = torch.randn(opt.batchSize, nz, 1, 1, device=device) + real_label = 1 + fake_label = 0 + + # setup optimizer + optimizerD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) + optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) + + for epoch in range(opt.niter): + for i, data in enumerate(dataloader, 0): + ############################ + # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z))) + ########################### + # train with real + netD.zero_grad() + real_cpu = data[0].to(device) + batch_size = real_cpu.size(0) + label = torch.full( + (batch_size,), real_label, dtype=real_cpu.dtype, device=device + ) + + output = netD(real_cpu) + errD_real = criterion(output, label) + errD_real.backward() + D_x = output.mean().item() + + # train with fake + noise = torch.randn(batch_size, nz, 1, 1, device=device) + fake = netG(noise) + label.fill_(fake_label) + output = netD(fake.detach()) + errD_fake = criterion(output, label) + errD_fake.backward() + D_G_z1 = output.mean().item() + errD = errD_real + errD_fake + optimizerD.step() + + ############################ + # (2) Update G network: maximize log(D(G(z))) + ########################### + netG.zero_grad() + label.fill_(real_label) # fake labels are real for generator cost + output = netD(fake) + errG = criterion(output, label) + errG.backward() + D_G_z2 = output.mean().item() + optimizerG.step() + + print( + "[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)): %.4f / %.4f" + % ( + epoch, + opt.niter, + i, + len(dataloader), + errD.item(), + errG.item(), + D_x, + D_G_z1, + D_G_z2, + ) + ) + if i % 100 == 0: + vutils.save_image( + real_cpu, "%s/real_samples.png" % opt.outf, normalize=True + ) + fake = netG(fixed_noise) + vutils.save_image( + fake.detach(), + "%s/fake_samples_epoch_%03d.png" % (opt.outf, epoch), + normalize=True, + ) + # do checkpointing + torch.save(netG.state_dict(), "%s/netG_epoch_%d.pth" % (opt.outf, epoch)) + torch.save(netD.state_dict(), "%s/netD_epoch_%d.pth" % (opt.outf, epoch)) + + +if __name__ == "__main__": + main() \ No newline at end of file From 3e4d58cbaa4e44fce586d0104e0507304c143470 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sun, 21 Mar 2021 23:57:48 +0100 Subject: [PATCH 005/331] mv --- pl_examples/{domain_templates => }/v3_example.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pl_examples/{domain_templates => }/v3_example.py (100%) diff --git a/pl_examples/domain_templates/v3_example.py b/pl_examples/v3_example.py similarity index 100% rename from pl_examples/domain_templates/v3_example.py rename to pl_examples/v3_example.py From 6201d95741b1df92aabba3d7a38a3a71b9311df4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 22 Mar 2021 00:00:24 +0100 Subject: [PATCH 006/331] set device --- pl_examples/v3_example.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pl_examples/v3_example.py b/pl_examples/v3_example.py index 46bc58d143db3..46ffa236058fc 100644 --- a/pl_examples/v3_example.py +++ b/pl_examples/v3_example.py @@ -40,6 +40,7 @@ parser.add_argument( "--outf", default="./lightning_logs", help="folder to output images and model checkpoints" ) +parser.add_argument("--local_rank", type=int, default=0) opt = parser.parse_args() os.makedirs(opt.outf, exist_ok=True) @@ -123,6 +124,8 @@ def main(): random.seed(123) torch.manual_seed(123) + torch.cuda.set_device(opt.local_rank) + dataset = dset.MNIST( root=".", download=True, From 38a01c4f6859b545e5eafcf5a424988078ca61d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 22 Mar 2021 00:01:49 +0100 Subject: [PATCH 007/331] set local rank --- pl_examples/v3_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/v3_example.py b/pl_examples/v3_example.py index 46ffa236058fc..a2a65ba4dae51 100644 --- a/pl_examples/v3_example.py +++ b/pl_examples/v3_example.py @@ -46,7 +46,7 @@ os.makedirs(opt.outf, exist_ok=True) nc = 1 -device = torch.device("cpu") +device = torch.device("cuda", index=opt.local_rank) ngpu = int(opt.ngpu) nz = int(opt.nz) ngf = int(opt.ngf) From 0faf6735480ebaf2c77819d3ed534e39d000d5cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 22 Mar 2021 00:26:59 +0100 Subject: [PATCH 008/331] fix circular import --- .../plugins/training_type/ddp.py | 3 +- pytorch_lightning/trainer/data_loading.py | 70 +----------------- pytorch_lightning/utilities/distributed.py | 71 ++++++++++++++++++- 3 files changed, 73 insertions(+), 71 deletions(-) diff --git a/pytorch_lightning/plugins/training_type/ddp.py b/pytorch_lightning/plugins/training_type/ddp.py index 48b7dbca20ff4..cfdadb97389d5 100644 --- a/pytorch_lightning/plugins/training_type/ddp.py +++ b/pytorch_lightning/plugins/training_type/ddp.py @@ -31,9 +31,8 @@ from pytorch_lightning.overrides.distributed import prepare_for_backward from pytorch_lightning.plugins.environments.cluster_environment import ClusterEnvironment from pytorch_lightning.plugins.training_type.parallel import ParallelPlugin -from pytorch_lightning.trainer.data_loading import replace_sampler from pytorch_lightning.utilities import _HYDRA_AVAILABLE, _TORCH_GREATER_EQUAL_1_7, rank_zero_warn -from pytorch_lightning.utilities.distributed import rank_zero_only, ReduceOp, sync_ddp_if_available +from pytorch_lightning.utilities.distributed import rank_zero_only, ReduceOp, sync_ddp_if_available, replace_sampler from pytorch_lightning.utilities.exceptions import MisconfigurationException from pytorch_lightning.utilities.seed import seed_everything diff --git a/pytorch_lightning/trainer/data_loading.py b/pytorch_lightning/trainer/data_loading.py index f1cdf40444ff4..836368941e9e5 100644 --- a/pytorch_lightning/trainer/data_loading.py +++ b/pytorch_lightning/trainer/data_loading.py @@ -11,14 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import inspect import multiprocessing import platform from abc import ABC from copy import deepcopy from typing import Callable, Iterable, List, Tuple, Union -from torch.utils.data import BatchSampler, DataLoader, RandomSampler, SequentialSampler, Sampler +from torch.utils.data import DataLoader, RandomSampler, SequentialSampler from torch.utils.data.distributed import DistributedSampler from pytorch_lightning.accelerators import Accelerator @@ -29,6 +28,7 @@ from pytorch_lightning.utilities.apply_func import apply_to_collection from pytorch_lightning.utilities.data import has_iterable_dataset, has_len from pytorch_lightning.utilities.debugging import InternalDebugger +from pytorch_lightning.utilities.distributed import replace_sampler from pytorch_lightning.utilities.exceptions import MisconfigurationException from pytorch_lightning.utilities.model_helpers import is_overridden @@ -337,69 +337,3 @@ def _flatten_dl_only(self, dataloaders): dataloaders = list(dataloaders) return dataloaders - - -def replace_sampler(dataloader: DataLoader, sampler: Sampler): - skip_keys = ('sampler', 'batch_sampler', 'dataset_kind') - skip_signature_keys = ('args', 'kwargs', 'self') - - attrs = {k: v for k, v in vars(dataloader).items() if not k.startswith("_")} - - params = set(inspect.signature(dataloader.__init__).parameters) - contains_dataset = True - - if type(dataloader) is not DataLoader: - contains_dataset = "dataset" in params - params.update(inspect.signature(DataLoader.__init__).parameters) - - dl_args = {name: attrs[name] for name in params if name in attrs and name not in skip_keys} - dl_args = _resolve_batch_sampler(dl_args, dataloader, sampler) - - multiprocessing_context = dataloader.multiprocessing_context - dl_args['multiprocessing_context'] = multiprocessing_context - - missing_kwargs = params.difference(skip_signature_keys).difference(dl_args) - if missing_kwargs: - """ - Example: - class CustomDataLoader(DataLoader): - def __init__(self, num_features, dataset, *args, **kwargs): - self.num_features = num_features - super().__init__(dataset, *args, **kwargs) - """ - dataloader_cls_name = dataloader.__class__.__name__ - raise MisconfigurationException( - f"Trying to inject DistributedSampler within {dataloader_cls_name} class." - "This would fail as your DataLoader doesn't expose all its __init__ parameters as attributes. " - f"Missing attributes are {missing_kwargs}. " - f"HINT: If you wrote the {dataloader_cls_name} class, add the `__init__` arguments as attributes or ", - "manually add DistributedSampler as " - f"{dataloader_cls_name}(dataset, ..., sampler=DistributedSampler(dataset, ...)).", - ) - - if not contains_dataset: - dl_args.pop('dataset') - - dataloader = type(dataloader)(**dl_args) - dataloader.multiprocessing_context = multiprocessing_context - return dataloader - - -def _resolve_batch_sampler(dl_args, dataloader, sampler): - batch_sampler = getattr(dataloader, "batch_sampler") - if batch_sampler is not None and type(batch_sampler) is not BatchSampler: - batch_sampler = type(batch_sampler)( - sampler, - batch_size=batch_sampler.batch_size, - drop_last=batch_sampler.drop_last, - ) - dl_args['batch_sampler'] = batch_sampler - dl_args['batch_size'] = 1 - dl_args['shuffle'] = False - dl_args['sampler'] = None - dl_args['drop_last'] = False - else: - dl_args['sampler'] = sampler - dl_args['shuffle'] = False - dl_args['batch_sampler'] = None - return dl_args diff --git a/pytorch_lightning/utilities/distributed.py b/pytorch_lightning/utilities/distributed.py index 658f349a22215..c6c988a160d93 100644 --- a/pytorch_lightning/utilities/distributed.py +++ b/pytorch_lightning/utilities/distributed.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import inspect import logging import os import warnings @@ -19,6 +19,9 @@ from typing import Any, Optional, Union import torch +from torch.utils.data import DataLoader, Sampler, BatchSampler + +from pytorch_lightning.utilities.exceptions import MisconfigurationException log = logging.getLogger(__name__) @@ -197,3 +200,69 @@ def all_gather_ddp_if_available( with torch.no_grad(): return AllGatherGrad.apply(tensor, group) return tensor + + +def replace_sampler(dataloader: DataLoader, sampler: Sampler): + skip_keys = ('sampler', 'batch_sampler', 'dataset_kind') + skip_signature_keys = ('args', 'kwargs', 'self') + + attrs = {k: v for k, v in vars(dataloader).items() if not k.startswith("_")} + + params = set(inspect.signature(dataloader.__init__).parameters) + contains_dataset = True + + if type(dataloader) is not DataLoader: + contains_dataset = "dataset" in params + params.update(inspect.signature(DataLoader.__init__).parameters) + + dl_args = {name: attrs[name] for name in params if name in attrs and name not in skip_keys} + dl_args = _resolve_batch_sampler(dl_args, dataloader, sampler) + + multiprocessing_context = dataloader.multiprocessing_context + dl_args['multiprocessing_context'] = multiprocessing_context + + missing_kwargs = params.difference(skip_signature_keys).difference(dl_args) + if missing_kwargs: + """ + Example: + class CustomDataLoader(DataLoader): + def __init__(self, num_features, dataset, *args, **kwargs): + self.num_features = num_features + super().__init__(dataset, *args, **kwargs) + """ + dataloader_cls_name = dataloader.__class__.__name__ + raise MisconfigurationException( + f"Trying to inject DistributedSampler within {dataloader_cls_name} class." + "This would fail as your DataLoader doesn't expose all its __init__ parameters as attributes. " + f"Missing attributes are {missing_kwargs}. " + f"HINT: If you wrote the {dataloader_cls_name} class, add the `__init__` arguments as attributes or ", + "manually add DistributedSampler as " + f"{dataloader_cls_name}(dataset, ..., sampler=DistributedSampler(dataset, ...)).", + ) + + if not contains_dataset: + dl_args.pop('dataset') + + dataloader = type(dataloader)(**dl_args) + dataloader.multiprocessing_context = multiprocessing_context + return dataloader + + +def _resolve_batch_sampler(dl_args, dataloader, sampler): + batch_sampler = getattr(dataloader, "batch_sampler") + if batch_sampler is not None and type(batch_sampler) is not BatchSampler: + batch_sampler = type(batch_sampler)( + sampler, + batch_size=batch_sampler.batch_size, + drop_last=batch_sampler.drop_last, + ) + dl_args['batch_sampler'] = batch_sampler + dl_args['batch_size'] = 1 + dl_args['shuffle'] = False + dl_args['sampler'] = None + dl_args['drop_last'] = False + else: + dl_args['sampler'] = sampler + dl_args['shuffle'] = False + dl_args['batch_sampler'] = None + return dl_args \ No newline at end of file From b04495dba10e64b700300d81971e54fa0a466dc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 22 Mar 2021 00:58:49 +0100 Subject: [PATCH 009/331] make example work --- pl_examples/accelerator_examples/__init__.py | 0 .../gan_example.py} | 117 ++++++------------ pl_examples/accelerator_examples/models.py | 76 ++++++++++++ .../accelerators/acceleratorV3.py | 14 ++- .../plugins/training_type/ddp.py | 4 +- 5 files changed, 126 insertions(+), 85 deletions(-) create mode 100644 pl_examples/accelerator_examples/__init__.py rename pl_examples/{v3_example.py => accelerator_examples/gan_example.py} (63%) create mode 100644 pl_examples/accelerator_examples/models.py diff --git a/pl_examples/accelerator_examples/__init__.py b/pl_examples/accelerator_examples/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/pl_examples/v3_example.py b/pl_examples/accelerator_examples/gan_example.py similarity index 63% rename from pl_examples/v3_example.py rename to pl_examples/accelerator_examples/gan_example.py index a2a65ba4dae51..902947a67b68d 100644 --- a/pl_examples/v3_example.py +++ b/pl_examples/accelerator_examples/gan_example.py @@ -1,3 +1,11 @@ +""" +DCGAN - Adapted from pytorch/examples + +Launch it with this command: + +python -m torch.distributed.launch --nproc_per_node=2 gan_example.py + +""" from __future__ import print_function import argparse import os @@ -10,6 +18,11 @@ import torchvision.datasets as dset import torchvision.transforms as transforms import torchvision.utils as vutils +from torch.nn.parallel import DistributedDataParallel +from torch.utils.data import DistributedSampler + +from pl_examples.accelerator_examples.models import weights_init, Generator, Discriminator +from pytorch_lightning.accelerators.acceleratorV3 import AcceleratorV3 parser = argparse.ArgumentParser() parser.add_argument( @@ -22,9 +35,6 @@ default=64, help="the height / width of the input image to network", ) -parser.add_argument("--nz", type=int, default=100, help="size of the latent z vector") -parser.add_argument("--ngf", type=int, default=64) -parser.add_argument("--ndf", type=int, default=64) parser.add_argument( "--niter", type=int, default=25, help="number of epochs to train for" ) @@ -45,86 +55,24 @@ opt = parser.parse_args() os.makedirs(opt.outf, exist_ok=True) -nc = 1 +# TODO: how do we handle this in Accelerator device = torch.device("cuda", index=opt.local_rank) ngpu = int(opt.ngpu) -nz = int(opt.nz) -ngf = int(opt.ngf) -ndf = int(opt.ndf) - - -# custom weights initialization called on netG and netD -def weights_init(m): - classname = m.__class__.__name__ - if classname.find("Conv") != -1: - torch.nn.init.normal_(m.weight, 0.0, 0.02) - elif classname.find("BatchNorm") != -1: - torch.nn.init.normal_(m.weight, 1.0, 0.02) - torch.nn.init.zeros_(m.bias) - - -class Generator(nn.Module): - def __init__(self): - super(Generator, self).__init__() - self.main = nn.Sequential( - # input is Z, going into a convolution - nn.ConvTranspose2d(nz, ngf * 8, 4, 1, 0, bias=False), - nn.BatchNorm2d(ngf * 8), - nn.ReLU(True), - # state size. (ngf*8) x 4 x 4 - nn.ConvTranspose2d(ngf * 8, ngf * 4, 4, 2, 1, bias=False), - nn.BatchNorm2d(ngf * 4), - nn.ReLU(True), - # state size. (ngf*4) x 8 x 8 - nn.ConvTranspose2d(ngf * 4, ngf * 2, 4, 2, 1, bias=False), - nn.BatchNorm2d(ngf * 2), - nn.ReLU(True), - # state size. (ngf*2) x 16 x 16 - nn.ConvTranspose2d(ngf * 2, ngf, 4, 2, 1, bias=False), - nn.BatchNorm2d(ngf), - nn.ReLU(True), - # state size. (ngf) x 32 x 32 - nn.ConvTranspose2d(ngf, nc, 4, 2, 1, bias=False), - nn.Tanh() - # state size. (nc) x 64 x 64 - ) - - def forward(self, input): - return self.main(input) - -class Discriminator(nn.Module): - def __init__(self): - super(Discriminator, self).__init__() - self.main = nn.Sequential( - # input is (nc) x 64 x 64 - nn.Conv2d(nc, ndf, 4, 2, 1, bias=False), - nn.LeakyReLU(0.2, inplace=True), - # state size. (ndf) x 32 x 32 - nn.Conv2d(ndf, ndf * 2, 4, 2, 1, bias=False), - nn.BatchNorm2d(ndf * 2), - nn.LeakyReLU(0.2, inplace=True), - # state size. (ndf*2) x 16 x 16 - nn.Conv2d(ndf * 2, ndf * 4, 4, 2, 1, bias=False), - nn.BatchNorm2d(ndf * 4), - nn.LeakyReLU(0.2, inplace=True), - # state size. (ndf*4) x 8 x 8 - nn.Conv2d(ndf * 4, ndf * 8, 4, 2, 1, bias=False), - nn.BatchNorm2d(ndf * 8), - nn.LeakyReLU(0.2, inplace=True), - # state size. (ndf*8) x 4 x 4 - nn.Conv2d(ndf * 8, 1, 4, 1, 0, bias=False), - nn.Sigmoid(), - ) - - def forward(self, input): - output = self.main(input) - return output.view(-1, 1).squeeze(1) + +nz = 100 + def main(): random.seed(123) torch.manual_seed(123) - torch.cuda.set_device(opt.local_rank) + # TODO: how do we handle this in Accelerator + # torch.cuda.set_device(opt.local_rank) + # TODO: how do we handle this? + os.environ["LOCAL_RANK"] = str(opt.local_rank) + # os.environ["NODE_RANK"] = str(opt.local_rank) + + accelerator = AcceleratorV3() dataset = dset.MNIST( root=".", @@ -141,12 +89,23 @@ def main(): dataset, batch_size=opt.batchSize, shuffle=True, num_workers=opt.workers ) - netG = Generator().to(device) + dataloader = accelerator.setup(dataloader) + assert isinstance(dataloader.sampler, DistributedSampler) + + netG = Generator() netG.apply(weights_init) - netD = Discriminator().to(device) + netD = Discriminator() netD.apply(weights_init) + accelerator.to_device(netG) + accelerator.to_device(netD) + + netG, netD = accelerator.setup(netG, netD) + + assert isinstance(netG, DistributedDataParallel) + assert isinstance(netD, DistributedDataParallel) + criterion = nn.BCELoss() fixed_noise = torch.randn(opt.batchSize, nz, 1, 1, device=device) @@ -157,6 +116,8 @@ def main(): optimizerD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) + optimizerG, optimizerG = accelerator.setup(optimizerG, optimizerD) + for epoch in range(opt.niter): for i, data in enumerate(dataloader, 0): ############################ diff --git a/pl_examples/accelerator_examples/models.py b/pl_examples/accelerator_examples/models.py new file mode 100644 index 0000000000000..70709e30b7b95 --- /dev/null +++ b/pl_examples/accelerator_examples/models.py @@ -0,0 +1,76 @@ +import torch +from torch import nn as nn + + +nc = 1 +nz = 100 +ngf = 64 +ndf = 64 + + +def weights_init(m): + classname = m.__class__.__name__ + if classname.find("Conv") != -1: + torch.nn.init.normal_(m.weight, 0.0, 0.02) + elif classname.find("BatchNorm") != -1: + torch.nn.init.normal_(m.weight, 1.0, 0.02) + torch.nn.init.zeros_(m.bias) + + +class Generator(nn.Module): + def __init__(self): + super(Generator, self).__init__() + self.main = nn.Sequential( + # input is Z, going into a convolution + nn.ConvTranspose2d(nz, ngf * 8, 4, 1, 0, bias=False), + nn.BatchNorm2d(ngf * 8), + nn.ReLU(True), + # state size. (ngf*8) x 4 x 4 + nn.ConvTranspose2d(ngf * 8, ngf * 4, 4, 2, 1, bias=False), + nn.BatchNorm2d(ngf * 4), + nn.ReLU(True), + # state size. (ngf*4) x 8 x 8 + nn.ConvTranspose2d(ngf * 4, ngf * 2, 4, 2, 1, bias=False), + nn.BatchNorm2d(ngf * 2), + nn.ReLU(True), + # state size. (ngf*2) x 16 x 16 + nn.ConvTranspose2d(ngf * 2, ngf, 4, 2, 1, bias=False), + nn.BatchNorm2d(ngf), + nn.ReLU(True), + # state size. (ngf) x 32 x 32 + nn.ConvTranspose2d(ngf, nc, 4, 2, 1, bias=False), + nn.Tanh() + # state size. (nc) x 64 x 64 + ) + + def forward(self, input): + return self.main(input) + + +class Discriminator(nn.Module): + def __init__(self): + super(Discriminator, self).__init__() + self.main = nn.Sequential( + # input is (nc) x 64 x 64 + nn.Conv2d(nc, ndf, 4, 2, 1, bias=False), + nn.LeakyReLU(0.2, inplace=True), + # state size. (ndf) x 32 x 32 + nn.Conv2d(ndf, ndf * 2, 4, 2, 1, bias=False), + nn.BatchNorm2d(ndf * 2), + nn.LeakyReLU(0.2, inplace=True), + # state size. (ndf*2) x 16 x 16 + nn.Conv2d(ndf * 2, ndf * 4, 4, 2, 1, bias=False), + nn.BatchNorm2d(ndf * 4), + nn.LeakyReLU(0.2, inplace=True), + # state size. (ndf*4) x 8 x 8 + nn.Conv2d(ndf * 4, ndf * 8, 4, 2, 1, bias=False), + nn.BatchNorm2d(ndf * 8), + nn.LeakyReLU(0.2, inplace=True), + # state size. (ndf*8) x 4 x 4 + nn.Conv2d(ndf * 8, 1, 4, 1, 0, bias=False), + nn.Sigmoid(), + ) + + def forward(self, input): + output = self.main(input) + return output.view(-1, 1).squeeze(1) \ No newline at end of file diff --git a/pytorch_lightning/accelerators/acceleratorV3.py b/pytorch_lightning/accelerators/acceleratorV3.py index 9fa20a0891639..eda097c21b448 100644 --- a/pytorch_lightning/accelerators/acceleratorV3.py +++ b/pytorch_lightning/accelerators/acceleratorV3.py @@ -15,10 +15,11 @@ class AcceleratedOptimizer(Optimizer): def __init__(self, optimizer: Optimizer): - super().__init__(params=optimizer.param_groups, default={}) # TODO: why is it called default and not defaults? + super().__init__(params=optimizer.param_groups, defaults={}) self.optimizer = optimizer - def step(self, closure: Optional[Callable[[], float]]=...) -> Optional[float]: + def step(self, closure=None): + # TODO: do precision magic here return self.optimizer.step(closure) @@ -45,11 +46,14 @@ def setup(self, *objects: Union[nn.Module, Optimizer, DataLoader]): wrapped_objects = [] for obj in objects: if isinstance(obj, nn.Module): - wrapped_objects.append(self.setup_model(obj)) + wrapped_objects.extend(self.setup_model(obj)) if isinstance(obj, Optimizer): - wrapped_objects.append(self.setup_optimizer(obj)) + wrapped_objects.extend(self.setup_optimizer(obj)) if isinstance(obj, DataLoader): - wrapped_objects.append(self.setup_dataloader(obj)) + wrapped_objects.extend(self.setup_dataloader(obj)) + + if len(wrapped_objects) == 1: + return wrapped_objects[0] return wrapped_objects def setup_model(self, *models: nn.Module): diff --git a/pytorch_lightning/plugins/training_type/ddp.py b/pytorch_lightning/plugins/training_type/ddp.py index cfdadb97389d5..cb7661445949b 100644 --- a/pytorch_lightning/plugins/training_type/ddp.py +++ b/pytorch_lightning/plugins/training_type/ddp.py @@ -95,7 +95,7 @@ def setup_environment(self): def setup_model(self, model: nn.Module) -> DistributedDataParallel: self.setup_distributed() # setup distributed if it is not already initialized model = DistributedDataParallel( - model, + model.to(self.root_device), device_ids=self.determine_ddp_device_ids(), **self._ddp_kwargs, ) @@ -218,7 +218,7 @@ def _check_can_spawn_children(self): ) def set_world_ranks(self): - self.local_rank = self.task_idx + self.local_rank = self.cluster_environment.local_rank() self.node_rank = self.cluster_environment.node_rank() self.global_rank = self.node_rank * self.num_processes + self.local_rank self.world_size = self.num_nodes * self.num_processes From abf2f4f47194f6083f9b8d2aec9827c9003b3669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 22 Mar 2021 19:07:15 +0100 Subject: [PATCH 010/331] automator and backend connector --- .../__init__.py | 0 .../gan_example.py | 18 ++-- .../models.py | 0 pytorch_lightning/automator/__init__.py | 0 .../automator.py} | 84 ++++++++++++++----- 5 files changed, 73 insertions(+), 29 deletions(-) rename pl_examples/{accelerator_examples => automator_examples}/__init__.py (100%) rename pl_examples/{accelerator_examples => automator_examples}/gan_example.py (93%) rename pl_examples/{accelerator_examples => automator_examples}/models.py (100%) create mode 100644 pytorch_lightning/automator/__init__.py rename pytorch_lightning/{accelerators/acceleratorV3.py => automator/automator.py} (58%) diff --git a/pl_examples/accelerator_examples/__init__.py b/pl_examples/automator_examples/__init__.py similarity index 100% rename from pl_examples/accelerator_examples/__init__.py rename to pl_examples/automator_examples/__init__.py diff --git a/pl_examples/accelerator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py similarity index 93% rename from pl_examples/accelerator_examples/gan_example.py rename to pl_examples/automator_examples/gan_example.py index 902947a67b68d..54998fa9ea2ba 100644 --- a/pl_examples/accelerator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -22,7 +22,7 @@ from torch.utils.data import DistributedSampler from pl_examples.accelerator_examples.models import weights_init, Generator, Discriminator -from pytorch_lightning.accelerators.acceleratorV3 import AcceleratorV3 +from pytorch_lightning.accelerators.acceleratorV3 import Automator parser = argparse.ArgumentParser() parser.add_argument( @@ -66,13 +66,13 @@ def main(): random.seed(123) torch.manual_seed(123) - # TODO: how do we handle this in Accelerator + # TODO: how do we handle this in Accelerator? # torch.cuda.set_device(opt.local_rank) # TODO: how do we handle this? os.environ["LOCAL_RANK"] = str(opt.local_rank) # os.environ["NODE_RANK"] = str(opt.local_rank) - accelerator = AcceleratorV3() + automator = Automator(gpus=2, accelerator="ddp") dataset = dset.MNIST( root=".", @@ -89,7 +89,7 @@ def main(): dataset, batch_size=opt.batchSize, shuffle=True, num_workers=opt.workers ) - dataloader = accelerator.setup(dataloader) + dataloader = automator.setup(dataloader) assert isinstance(dataloader.sampler, DistributedSampler) netG = Generator() @@ -98,10 +98,10 @@ def main(): netD = Discriminator() netD.apply(weights_init) - accelerator.to_device(netG) - accelerator.to_device(netD) + automator.to_device(netG) + automator.to_device(netD) - netG, netD = accelerator.setup(netG, netD) + netG, netD = automator.setup(netG, netD) assert isinstance(netG, DistributedDataParallel) assert isinstance(netD, DistributedDataParallel) @@ -116,7 +116,7 @@ def main(): optimizerD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) - optimizerG, optimizerG = accelerator.setup(optimizerG, optimizerD) + optimizerG, optimizerG = automator.setup(optimizerG, optimizerD) for epoch in range(opt.niter): for i, data in enumerate(dataloader, 0): @@ -188,4 +188,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/pl_examples/accelerator_examples/models.py b/pl_examples/automator_examples/models.py similarity index 100% rename from pl_examples/accelerator_examples/models.py rename to pl_examples/automator_examples/models.py diff --git a/pytorch_lightning/automator/__init__.py b/pytorch_lightning/automator/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/pytorch_lightning/accelerators/acceleratorV3.py b/pytorch_lightning/automator/automator.py similarity index 58% rename from pytorch_lightning/accelerators/acceleratorV3.py rename to pytorch_lightning/automator/automator.py index eda097c21b448..3bd0bbae906a4 100644 --- a/pytorch_lightning/accelerators/acceleratorV3.py +++ b/pytorch_lightning/automator/automator.py @@ -1,19 +1,18 @@ from collections import Callable -from typing import Any, Union, Optional +from contextlib import contextmanager +from typing import Any, Union -import torch import torch.nn as nn from torch import Tensor from torch.optim import Optimizer from torch.utils.data import DataLoader - -from pytorch_lightning.plugins import DDPPlugin, PrecisionPlugin -from pytorch_lightning.plugins.environments import LightningEnvironment +from pytorch_lightning.trainer.connectors.accelerator_connector import ( + AcceleratorConnector, +) from pytorch_lightning.utilities import move_data_to_device -class AcceleratedOptimizer(Optimizer): - +class AutomatedOptimizer(Optimizer): def __init__(self, optimizer: Optimizer): super().__init__(params=optimizer.param_groups, defaults={}) self.optimizer = optimizer @@ -23,18 +22,45 @@ def step(self, closure=None): return self.optimizer.step(closure) -class AcceleratorV3: - - def __init__(self): - # hardcoded for a start - # this also needs to incorporate some of the accelerator connectors logic for argument handling - self.training_type_plugin = DDPPlugin( - parallel_devices=[torch.device("cuda", 0), torch.device("cuda", 1)], - num_nodes=1, - cluster_environment=LightningEnvironment(), +class Automator: + def __init__( + self, + accelerator=None, + plugin=None, + gpus=None, + tpus=None, + num_processes=None, + num_nodes=1, + precision=32, + amp_backend: str = "native", + amp_level: str = "O2", + ): + backend_connector = AcceleratorConnector( + gpus=gpus, + tpu_cores=tpus, + num_processes=num_processes, + distributed_backend=accelerator, + num_nodes=num_nodes, + precision=precision, + amp_type=amp_backend, + amp_level=amp_level, + plugins=[plugin], + # TODO: + deterministic=False, sync_batchnorm=False, + benchmark=False, + replace_sampler_ddp=True, + auto_select_gpus=False, ) - self.precision_plugin = PrecisionPlugin() + self.accelerator = backend_connector.select_accelerator() + + @property + def training_type_plugin(self): + return self.accelerator.training_type_plugin + + @property + def precision_plugin(self): + return self.accelerator.precision_plugin @property def device(self): @@ -63,16 +89,34 @@ def setup_model(self, *models: nn.Module): def setup_optimizer(self, *optimizers: Optimizer): # user can call this method independently instead of the general purpose setup method # TODO: let plugin setup optimizer too? - return [AcceleratedOptimizer(optimizer) for optimizer in optimizers] + return [AutomatedOptimizer(optimizer) for optimizer in optimizers] def setup_dataloader(self, *dataloaders: DataLoader): # user can call this method independently instead of the general purpose setup method - return [self.training_type_plugin.setup_dataloader(dataloader) for dataloader in dataloaders] + return [ + self.training_type_plugin.setup_dataloader(dataloader) + for dataloader in dataloaders + ] def backward(self, tensor: Tensor, *args, **kwargs): # TODO: precision plugin backward return tensor.backward(*args, **kwargs) + @contextmanager + def forward_context(self): + # basically only for autocast and block ddp sync + yield + + @contextmanager + def backward_context(self, *args, **kwargs): + # necessary for deepspeed backward + scaler in AMP + yield + + @contextmanager + def optimizer_step_context(self, *args, **kwargs): + # necessary for deepspeed + scaling + yield + def to_device(self, obj: Union[nn.Module, Tensor]) -> Union[nn.Module, Tensor]: if isinstance(obj, nn.Module): return obj.to(self.device) @@ -94,4 +138,4 @@ def save_checkpoint(self, filepath): pass def execute_on_rank(self, func: Callable, rank: int): - pass \ No newline at end of file + pass From dc5015455149c3eb3eaf001d7476fc5dcd963e94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 22 Mar 2021 19:09:35 +0100 Subject: [PATCH 011/331] update imports --- pl_examples/automator_examples/gan_example.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index 54998fa9ea2ba..a39452354f9ee 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -21,8 +21,8 @@ from torch.nn.parallel import DistributedDataParallel from torch.utils.data import DistributedSampler -from pl_examples.accelerator_examples.models import weights_init, Generator, Discriminator -from pytorch_lightning.accelerators.acceleratorV3 import Automator +from pl_examples.automator_examples.models import weights_init, Generator, Discriminator +from pytorch_lightning.automator.automator import Automator parser = argparse.ArgumentParser() parser.add_argument( From bc088aa86f33f432a772dcd8c61273e949171665 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 22 Mar 2021 19:14:04 +0100 Subject: [PATCH 012/331] fix device and plugin arguments --- pl_examples/automator_examples/gan_example.py | 19 ++++++++----------- pytorch_lightning/automator/automator.py | 4 ++-- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index a39452354f9ee..8c3cc510e419e 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -54,9 +54,6 @@ opt = parser.parse_args() os.makedirs(opt.outf, exist_ok=True) - -# TODO: how do we handle this in Accelerator -device = torch.device("cuda", index=opt.local_rank) ngpu = int(opt.ngpu) nz = 100 @@ -72,7 +69,7 @@ def main(): os.environ["LOCAL_RANK"] = str(opt.local_rank) # os.environ["NODE_RANK"] = str(opt.local_rank) - automator = Automator(gpus=2, accelerator="ddp") + automator = Automator() dataset = dset.MNIST( root=".", @@ -90,7 +87,7 @@ def main(): ) dataloader = automator.setup(dataloader) - assert isinstance(dataloader.sampler, DistributedSampler) + # assert isinstance(dataloader.sampler, DistributedSampler) netG = Generator() netG.apply(weights_init) @@ -103,12 +100,12 @@ def main(): netG, netD = automator.setup(netG, netD) - assert isinstance(netG, DistributedDataParallel) - assert isinstance(netD, DistributedDataParallel) + # assert isinstance(netG, DistributedDataParallel) + # assert isinstance(netD, DistributedDataParallel) criterion = nn.BCELoss() - fixed_noise = torch.randn(opt.batchSize, nz, 1, 1, device=device) + fixed_noise = torch.randn(opt.batchSize, nz, 1, 1, device=automator.device) real_label = 1 fake_label = 0 @@ -125,10 +122,10 @@ def main(): ########################### # train with real netD.zero_grad() - real_cpu = data[0].to(device) + real_cpu = automator.to_device(data[0]) batch_size = real_cpu.size(0) label = torch.full( - (batch_size,), real_label, dtype=real_cpu.dtype, device=device + (batch_size,), real_label, dtype=real_cpu.dtype, device=automator.device ) output = netD(real_cpu) @@ -137,7 +134,7 @@ def main(): D_x = output.mean().item() # train with fake - noise = torch.randn(batch_size, nz, 1, 1, device=device) + noise = torch.randn(batch_size, nz, 1, 1, device=automator.device) fake = netG(noise) label.fill_(fake_label) output = netD(fake.detach()) diff --git a/pytorch_lightning/automator/automator.py b/pytorch_lightning/automator/automator.py index 3bd0bbae906a4..602941d1804e9 100644 --- a/pytorch_lightning/automator/automator.py +++ b/pytorch_lightning/automator/automator.py @@ -29,7 +29,7 @@ def __init__( plugin=None, gpus=None, tpus=None, - num_processes=None, + num_processes=1, num_nodes=1, precision=32, amp_backend: str = "native", @@ -44,7 +44,7 @@ def __init__( precision=precision, amp_type=amp_backend, amp_level=amp_level, - plugins=[plugin], + plugins=plugin, # TODO: deterministic=False, sync_batchnorm=False, From ea5c537d42ca1b76f4ded836ecda697ec34805d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 22 Mar 2021 19:16:15 +0100 Subject: [PATCH 013/331] example --- pl_examples/automator_examples/gan_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index 8c3cc510e419e..0a286044516ec 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -69,7 +69,7 @@ def main(): os.environ["LOCAL_RANK"] = str(opt.local_rank) # os.environ["NODE_RANK"] = str(opt.local_rank) - automator = Automator() + automator = Automator(gpus=2, accelerator="ddp") dataset = dset.MNIST( root=".", From fe5285028b444ac70d9ccf73cf38ff7645529385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 22 Mar 2021 23:47:21 +0100 Subject: [PATCH 014/331] forward context --- pytorch_lightning/accelerators/accelerator.py | 21 +++++++++++++++---- pytorch_lightning/plugins/base_plugin.py | 18 ++-------------- .../plugins/precision/native_amp.py | 20 +----------------- .../plugins/training_type/ddp.py | 8 ------- .../training_type/training_type_plugin.py | 17 +++++++++++++++ 5 files changed, 37 insertions(+), 47 deletions(-) diff --git a/pytorch_lightning/accelerators/accelerator.py b/pytorch_lightning/accelerators/accelerator.py index ceb9d98505acc..64f48df429133 100644 --- a/pytorch_lightning/accelerators/accelerator.py +++ b/pytorch_lightning/accelerators/accelerator.py @@ -14,6 +14,7 @@ from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, TYPE_CHECKING, Union import torch +import torch.nn as nn from torch.optim import Optimizer from torch.utils.data import DataLoader @@ -88,6 +89,12 @@ def setup(self, trainer: 'Trainer', model: LightningModule) -> None: self.setup_optimizers(trainer) self.setup_precision_plugin(self.precision_plugin) + def setup_dataloader(self, dataloader: DataLoader) -> DataLoader: + return self.training_type_plugin.setup_dataloader(dataloader) + + def setup_model(self, model: nn.Module) -> nn.Module: + return self.training_type_plugin.setup_model(model) + def start_training(self, trainer: 'Trainer') -> None: self.training_type_plugin.start_training(trainer) @@ -131,6 +138,12 @@ def lightning_module(self) -> LightningModule: def root_device(self) -> torch.device: return self.training_type_plugin.root_device + def wrap_model(self, model: nn.Module) -> nn.Module: + return self.training_type_plugin.wrap_model(model) + + def unwrap_model(self, model: nn.Module) -> nn.Module: + return self.training_type_plugin.unwrap_model(model) + def teardown(self) -> None: """This method is called to teardown the training process. It is the right place to release memory and free other ressources. @@ -174,7 +187,7 @@ def training_step( """ args[0] = self.to_device(args[0]) - with self.precision_plugin.train_step_context(), self.training_type_plugin.train_step_context(): + with self.precision_plugin.forward_context(), self.training_type_plugin.forward_context(): return self.training_type_plugin.training_step(*args) def post_training_step(self) -> None: @@ -195,7 +208,7 @@ def validation_step(self, args: List[Union[Any, int]]) -> _STEP_OUTPUT_TYPE: args[0] = batch - with self.precision_plugin.val_step_context(), self.training_type_plugin.val_step_context(): + with self.precision_plugin.forward_context(), self.training_type_plugin.forward_context(): return self.training_type_plugin.validation_step(*args) def test_step(self, args: List[Union[Any, int]]) -> _STEP_OUTPUT_TYPE: @@ -213,7 +226,7 @@ def test_step(self, args: List[Union[Any, int]]) -> _STEP_OUTPUT_TYPE: args[0] = batch - with self.precision_plugin.test_step_context(), self.training_type_plugin.test_step_context(): + with self.precision_plugin.forward_context(), self.training_type_plugin.forward_context(): return self.training_type_plugin.test_step(*args) def predict(self, args: List[Union[Any, int]]) -> _STEP_OUTPUT_TYPE: @@ -231,7 +244,7 @@ def predict(self, args: List[Union[Any, int]]) -> _STEP_OUTPUT_TYPE: args[0] = batch - with self.precision_plugin.predict_context(), self.training_type_plugin.predict_context(): + with self.precision_plugin.forward_context(), self.training_type_plugin.forward_context(): return self.training_type_plugin.predict(*args) def training_step_end(self, output: _STEP_OUTPUT_TYPE) -> _STEP_OUTPUT_TYPE: diff --git a/pytorch_lightning/plugins/base_plugin.py b/pytorch_lightning/plugins/base_plugin.py index f89bdf7a8aa72..d2e7a566d0d75 100644 --- a/pytorch_lightning/plugins/base_plugin.py +++ b/pytorch_lightning/plugins/base_plugin.py @@ -26,21 +26,7 @@ def post_dispatch(self) -> None: """Hook to do something after the training/evaluation/prediction finishes.""" @contextlib.contextmanager - def train_step_context(self) -> Generator: - """A contextmanager for the trainstep""" + def forward_context(self) -> Generator: + """A contextmanager for managing model forward/training_step/evaluation_step/predict_step""" yield - @contextlib.contextmanager - def val_step_context(self) -> Generator: - """A contextmanager for the validation step""" - yield - - @contextlib.contextmanager - def test_step_context(self) -> Generator: - """A contextmanager for the teststep""" - yield - - @contextlib.contextmanager - def predict_context(self) -> Generator: - """A contextmanager for the predict step""" - yield diff --git a/pytorch_lightning/plugins/precision/native_amp.py b/pytorch_lightning/plugins/precision/native_amp.py index 3c83945c8a1b7..864360ba7df46 100644 --- a/pytorch_lightning/plugins/precision/native_amp.py +++ b/pytorch_lightning/plugins/precision/native_amp.py @@ -99,25 +99,7 @@ def post_optimizer_step(self, optimizer: 'Optimizer', optimizer_idx: int) -> Non self.scaler.update() @contextmanager - def train_step_context(self) -> Generator[None, None, None]: - """Enable autocast context""" - with torch.cuda.amp.autocast(): - yield - - @contextmanager - def val_step_context(self) -> Generator[None, None, None]: - """Enable autocast context""" - with torch.cuda.amp.autocast(): - yield - - @contextmanager - def test_step_context(self) -> Generator[None, None, None]: - """Enable autocast context""" - with torch.cuda.amp.autocast(): - yield - - @contextmanager - def predict_context(self) -> Generator[None, None, None]: + def forward_context(self) -> Generator[None, None, None]: """Enable autocast context""" with torch.cuda.amp.autocast(): yield diff --git a/pytorch_lightning/plugins/training_type/ddp.py b/pytorch_lightning/plugins/training_type/ddp.py index cb7661445949b..0e408d942a19d 100644 --- a/pytorch_lightning/plugins/training_type/ddp.py +++ b/pytorch_lightning/plugins/training_type/ddp.py @@ -112,7 +112,6 @@ def _call_children_scripts(self): # bookkeeping of spawned processes assert self.global_rank == 0 - self._check_can_spawn_children() self._has_spawned_children = True # DDP Environment variables @@ -210,13 +209,6 @@ def setup_distributed(self): self.dist.rank = self.global_rank self.dist.device = self.root_device - def _check_can_spawn_children(self): - if self._has_spawned_children: - raise RuntimeError( - "You tried to run `.fit` or `.test` multiple times in the same script." - " This is not supported in DDP mode, switch to `distributed_backend='ddp_spawn'` instead." - ) - def set_world_ranks(self): self.local_rank = self.cluster_environment.local_rank() self.node_rank = self.cluster_environment.node_rank() diff --git a/pytorch_lightning/plugins/training_type/training_type_plugin.py b/pytorch_lightning/plugins/training_type/training_type_plugin.py index 800d7e3f1038d..9bdc6390b6873 100644 --- a/pytorch_lightning/plugins/training_type/training_type_plugin.py +++ b/pytorch_lightning/plugins/training_type/training_type_plugin.py @@ -28,6 +28,17 @@ from pytorch_lightning.trainer.trainer import Trainer +def call_hook(model: nn.Module, name: str, *args, **kwargs) -> Any: + """ + Call a hook on the model if it is available. + + Return: + None if hook is undefined, and the result of the hook otherwise. + """ + if hasattr(model, name) and callable(model.name): + return getattr(model, name)(*args, **kwargs) + + class TrainingTypePlugin(Plugin, ABC): """A Plugin to change the behaviour of the training, validation and test-loop.""" @@ -54,6 +65,12 @@ def setup_dataloader(self, dataloader: DataLoader) -> DataLoader: return dataloader def setup_model(self, model: nn.Module) -> nn.Module: + return self.wrap_model(model) + + def wrap_model(self, model: nn.Module) -> nn.Module: + return model + + def unwrap_model(self, model: nn.Module) -> nn.Module: return model @property From e28b483c9041d7d80fc26bf040e0046439d3b256 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 24 Mar 2021 00:40:33 +0100 Subject: [PATCH 015/331] optimizer step --- pl_examples/automator_examples/gan_example.py | 6 +- pytorch_lightning/accelerators/accelerator.py | 8 ++- pytorch_lightning/automator/automator.py | 55 ++++++++++++------- .../plugins/precision/native_amp.py | 6 ++ .../plugins/precision/precision_plugin.py | 6 ++ .../plugins/training_type/tpu_spawn.py | 6 +- 6 files changed, 60 insertions(+), 27 deletions(-) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index 0a286044516ec..0b4afbecc0146 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -130,7 +130,7 @@ def main(): output = netD(real_cpu) errD_real = criterion(output, label) - errD_real.backward() + automator.backward(errD_real) D_x = output.mean().item() # train with fake @@ -139,7 +139,7 @@ def main(): label.fill_(fake_label) output = netD(fake.detach()) errD_fake = criterion(output, label) - errD_fake.backward() + automator.backward(errD_fake) D_G_z1 = output.mean().item() errD = errD_real + errD_fake optimizerD.step() @@ -151,7 +151,7 @@ def main(): label.fill_(real_label) # fake labels are real for generator cost output = netD(fake) errG = criterion(output, label) - errG.backward() + automator.backward(errG) D_G_z2 = output.mean().item() optimizerG.step() diff --git a/pytorch_lightning/accelerators/accelerator.py b/pytorch_lightning/accelerators/accelerator.py index 64f48df429133..24098219f9ba9 100644 --- a/pytorch_lightning/accelerators/accelerator.py +++ b/pytorch_lightning/accelerators/accelerator.py @@ -309,14 +309,18 @@ def optimizer_step(self, optimizer: Optimizer, opt_idx: int, lambda_closure: Cal self.lightning_module, optimizer, opt_idx, lambda_closure, **kwargs ) if make_optimizer_step: - self.run_optimizer_step(optimizer, opt_idx, lambda_closure, **kwargs) + self.training_type_plugin.optimizer_step(optimizer, lambda_closure=lambda_closure, **kwargs) self.precision_plugin.post_optimizer_step(optimizer, opt_idx) self.training_type_plugin.post_optimizer_step(optimizer, opt_idx, **kwargs) def run_optimizer_step( - self, optimizer: Optimizer, optimizer_idx: int, lambda_closure: Callable, **kwargs: Any + self, optimizer: Optimizer, lambda_closure: Callable, **kwargs: Any ) -> None: + """Lightning-independent optimizer step logic""" + self.precision_plugin.run_pre_optimizer_step(optimizer) self.training_type_plugin.optimizer_step(optimizer, lambda_closure=lambda_closure, **kwargs) + self.precision_plugin.run_post_optimizer_step(optimizer) + # TODO: do we need to call training_type_plugin.post_optimizer_step? def optimizer_zero_grad(self, current_epoch: int, batch_idx: int, optimizer: Optimizer, opt_idx: int) -> None: """Zeros all model parameter's gradients""" diff --git a/pytorch_lightning/automator/automator.py b/pytorch_lightning/automator/automator.py index 602941d1804e9..b7e52440e8bb1 100644 --- a/pytorch_lightning/automator/automator.py +++ b/pytorch_lightning/automator/automator.py @@ -1,11 +1,14 @@ +from weakref import proxy from collections import Callable from contextlib import contextmanager -from typing import Any, Union +from typing import Any, Union, Optional import torch.nn as nn from torch import Tensor from torch.optim import Optimizer from torch.utils.data import DataLoader + +from pytorch_lightning.accelerators import Accelerator from pytorch_lightning.trainer.connectors.accelerator_connector import ( AcceleratorConnector, ) @@ -13,13 +16,21 @@ class AutomatedOptimizer(Optimizer): - def __init__(self, optimizer: Optimizer): + def __init__(self, optimizer: Optimizer, accelerator: Accelerator): super().__init__(params=optimizer.param_groups, defaults={}) self.optimizer = optimizer + self._accelerator = accelerator - def step(self, closure=None): + def step(self, closure=None, **kwargs: Any): # TODO: do precision magic here - return self.optimizer.step(closure) + print("running automated step") + output = self._accelerator.run_optimizer_step( + self.optimizer, + optimizer_idx=0, # TODO: remove optimizer_idx + lambda_closure=closure, + **kwargs, + ) + return output class Automator: @@ -89,33 +100,34 @@ def setup_model(self, *models: nn.Module): def setup_optimizer(self, *optimizers: Optimizer): # user can call this method independently instead of the general purpose setup method # TODO: let plugin setup optimizer too? - return [AutomatedOptimizer(optimizer) for optimizer in optimizers] + return [AutomatedOptimizer(optimizer=optimizer, accelerator=self.accelerator) for optimizer in optimizers] def setup_dataloader(self, *dataloaders: DataLoader): # user can call this method independently instead of the general purpose setup method - return [ + dataloaders = [ self.training_type_plugin.setup_dataloader(dataloader) for dataloader in dataloaders ] + return dataloaders def backward(self, tensor: Tensor, *args, **kwargs): # TODO: precision plugin backward + # self.precision_plugin.backward() return tensor.backward(*args, **kwargs) @contextmanager def forward_context(self): - # basically only for autocast and block ddp sync - yield - - @contextmanager - def backward_context(self, *args, **kwargs): - # necessary for deepspeed backward + scaler in AMP - yield - - @contextmanager - def optimizer_step_context(self, *args, **kwargs): - # necessary for deepspeed + scaling - yield + with self.precision_plugin.forward_context(), self.training_type_plugin.forward_context(): + yield + + # @contextmanager + # def backward_context(self, *args, **kwargs): + # yield + # + # @contextmanager + # def optimizer_step_context(self, *args, **kwargs): + # # necessary for deepspeed + scaling + # yield def to_device(self, obj: Union[nn.Module, Tensor]) -> Union[nn.Module, Tensor]: if isinstance(obj, nn.Module): @@ -126,12 +138,13 @@ def sync(self, data: Any) -> Any: pass def reduce_data(self, data: Any) -> Any: - pass + self.training_type_plugin.reduce(data) - def reduce_decision(self, decision: bool): - return False + def reduce_decision(self, decision: bool) -> bool: + return self.training_type_plugin.reduce_boolean_decision(decision) def broadcast_decision(self, decision: bool): + # return self.training_type_plugin.broadcast_boolean_decision(decision) return False def save_checkpoint(self, filepath): diff --git a/pytorch_lightning/plugins/precision/native_amp.py b/pytorch_lightning/plugins/precision/native_amp.py index 864360ba7df46..182f2673c7f39 100644 --- a/pytorch_lightning/plugins/precision/native_amp.py +++ b/pytorch_lightning/plugins/precision/native_amp.py @@ -95,6 +95,12 @@ def pre_optimizer_step( def post_optimizer_step(self, optimizer: 'Optimizer', optimizer_idx: int) -> None: """Updates the GradScaler""" + self.run_post_optimizer_step(optimizer) + + def run_pre_optimizer_step(self, optimizer: "Optimizer") -> None: + self.scaler.unscale_(optimizer) + + def run_post_optimizer_step(self, optimizer: "Optimizer") -> None: self.scaler.step(optimizer) self.scaler.update() diff --git a/pytorch_lightning/plugins/precision/precision_plugin.py b/pytorch_lightning/plugins/precision/precision_plugin.py index 7172d82391bd3..b42129c0472db 100644 --- a/pytorch_lightning/plugins/precision/precision_plugin.py +++ b/pytorch_lightning/plugins/precision/precision_plugin.py @@ -98,6 +98,12 @@ def pre_optimizer_step( def post_optimizer_step(self, optimizer: 'Optimizer', optimizer_idx: int) -> None: """Hook to do something after each optimizer step.""" + def run_pre_optimizer_step(self, optimizer: "Optimizer"): + """ Lightning-independent pre optimizer step logic. """ + + def run_post_optimizer_step(self, optimizer: "Optimizer"): + """ Lightning-independent post optimizer step logic. """ + def clip_gradients(self, optimizer: 'Optimizer', clip_val: Union[int, float], norm_type: float = 2.0) -> None: """Clips the gradients to a specific value""" if clip_val is None: diff --git a/pytorch_lightning/plugins/training_type/tpu_spawn.py b/pytorch_lightning/plugins/training_type/tpu_spawn.py index c883ff504f24d..6837f4c45dc70 100644 --- a/pytorch_lightning/plugins/training_type/tpu_spawn.py +++ b/pytorch_lightning/plugins/training_type/tpu_spawn.py @@ -14,11 +14,12 @@ import io import os import re -from typing import Any, Dict, Iterable, List, Optional, Union +from typing import Any, Dict, Iterable, List, Optional, Union, Callable import torch import torch.distributed as torch_distrib import torch.multiprocessing as mp +from torch.optim import Optimizer from pytorch_lightning.core.lightning import LightningModule from pytorch_lightning.plugins.training_type.ddp_spawn import DDPSpawnPlugin @@ -271,6 +272,9 @@ def xmp_spawn_kwargs(self): "start_method": self.start_method } + def optimizer_step(self, optimizer: Optimizer, lambda_closure: Callable, **kwargs): + xm.optimizer_step(optimizer, barrier=False, optimizer_args={'closure': lambda_closure, **kwargs}) + def start_training(self, trainer) -> None: # todo: precision pluging is call in accelerator setup and should be moved if 'XLA_USE_BF16' in os.environ: From 62896542b14a410834b0b23f7187d8b72385a12a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 24 Mar 2021 00:46:01 +0100 Subject: [PATCH 016/331] step --- pytorch_lightning/automator/automator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pytorch_lightning/automator/automator.py b/pytorch_lightning/automator/automator.py index b7e52440e8bb1..c492dffd1b7b2 100644 --- a/pytorch_lightning/automator/automator.py +++ b/pytorch_lightning/automator/automator.py @@ -26,7 +26,6 @@ def step(self, closure=None, **kwargs: Any): print("running automated step") output = self._accelerator.run_optimizer_step( self.optimizer, - optimizer_idx=0, # TODO: remove optimizer_idx lambda_closure=closure, **kwargs, ) From 810ea3bef9b63274b38c08148d325ae1ae80a0c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 24 Mar 2021 01:06:20 +0100 Subject: [PATCH 017/331] precision backward --- pytorch_lightning/accelerators/accelerator.py | 7 +++++++ pytorch_lightning/automator/automator.py | 4 +--- .../plugins/precision/deepspeed_precision.py | 4 ++++ pytorch_lightning/plugins/precision/native_amp.py | 8 +++++++- pytorch_lightning/plugins/precision/precision_plugin.py | 6 +++++- 5 files changed, 24 insertions(+), 5 deletions(-) diff --git a/pytorch_lightning/accelerators/accelerator.py b/pytorch_lightning/accelerators/accelerator.py index 24098219f9ba9..d9af7e04c5a33 100644 --- a/pytorch_lightning/accelerators/accelerator.py +++ b/pytorch_lightning/accelerators/accelerator.py @@ -15,6 +15,7 @@ import torch import torch.nn as nn +from torch import Tensor from torch.optim import Optimizer from torch.utils.data import DataLoader @@ -296,6 +297,12 @@ def backward( return output + def run_backward(self, tensor: Tensor, *args, **kwargs) -> None: + """ Lightning-independent backward logic """ + # TODO: Q: We don't need training_type.pre_/post_backward here right? Because we can't automate + # the blocking of "require_backward_grad_sync" for the PyTorch user + self.precision_plugin.run_backward(tensor, *args, **kwargs) + def optimizer_step(self, optimizer: Optimizer, opt_idx: int, lambda_closure: Callable, **kwargs: Any) -> None: """performs the actual optimizer step. diff --git a/pytorch_lightning/automator/automator.py b/pytorch_lightning/automator/automator.py index c492dffd1b7b2..b3378add03647 100644 --- a/pytorch_lightning/automator/automator.py +++ b/pytorch_lightning/automator/automator.py @@ -110,9 +110,7 @@ def setup_dataloader(self, *dataloaders: DataLoader): return dataloaders def backward(self, tensor: Tensor, *args, **kwargs): - # TODO: precision plugin backward - # self.precision_plugin.backward() - return tensor.backward(*args, **kwargs) + self.accelerator.run_backward(tensor, *args, **kwargs) @contextmanager def forward_context(self): diff --git a/pytorch_lightning/plugins/precision/deepspeed_precision.py b/pytorch_lightning/plugins/precision/deepspeed_precision.py index 6bcbb5ad851dc..bbbbf0e0505d2 100644 --- a/pytorch_lightning/plugins/precision/deepspeed_precision.py +++ b/pytorch_lightning/plugins/precision/deepspeed_precision.py @@ -75,6 +75,10 @@ def backward( return closure_loss + def run_backward(self, tensor, *args, **kwargs): + # TODO: implement + pass + def clip_gradients(self, optimizer: 'Optimizer', clip_val: Union[int, float], norm_type: float = 2.0) -> None: """ DeepSpeed handles clipping gradients via the training type plugin. diff --git a/pytorch_lightning/plugins/precision/native_amp.py b/pytorch_lightning/plugins/precision/native_amp.py index 182f2673c7f39..c9df56d5675ab 100644 --- a/pytorch_lightning/plugins/precision/native_amp.py +++ b/pytorch_lightning/plugins/precision/native_amp.py @@ -49,7 +49,7 @@ def backward( *args: Any, **kwargs: Any, ) -> torch.Tensor: - """performs the actual backpropagation + """Performs the actual backpropagation Args: model: the model to be optimized @@ -69,6 +69,12 @@ def backward( return closure_loss + def run_backward(self, tensor, *args, **kwargs): + tensor = self.scaler.scale(tensor) + super().run_backward(tensor, *args, **kwargs) + # self.scaler.unscale_(optimizer) # TODO: needed? + + def pre_optimizer_step( self, pl_module: 'LightningModule', diff --git a/pytorch_lightning/plugins/precision/precision_plugin.py b/pytorch_lightning/plugins/precision/precision_plugin.py index b42129c0472db..c18f7a5059ddc 100644 --- a/pytorch_lightning/plugins/precision/precision_plugin.py +++ b/pytorch_lightning/plugins/precision/precision_plugin.py @@ -77,13 +77,17 @@ def backward( if automatic_optimization: model.backward(closure_loss, optimizer, opt_idx) else: - closure_loss.backward(*args, **kwargs) + self.run_backward(closure_loss, *args, **kwargs) # once backward has been applied, release graph closure_loss = closure_loss.detach() return closure_loss + def run_backward(self, tensor, *args, **kwargs) -> None: + """ Lightning-independent backward logic. """ + tensor.backward(*args, **kwargs) + def pre_optimizer_step( self, pl_module: 'LightningModule', From 2f198b10d6c82f4cb4142ad5280781f68176f1c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 24 Mar 2021 01:16:21 +0100 Subject: [PATCH 018/331] update gan example precision --- pl_examples/automator_examples/gan_example.py | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index 0b4afbecc0146..d38c4b5cfd6aa 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -3,7 +3,8 @@ Launch it with this command: -python -m torch.distributed.launch --nproc_per_node=2 gan_example.py +python -m torch.distributed.launch --nproc_per_node=2 gan_example.py (--accelerator ddp --gpus 2 \ + --precision 16 ) """ from __future__ import print_function @@ -44,17 +45,28 @@ parser.add_argument( "--beta1", type=float, default=0.5, help="beta1 for adam. default=0.5" ) -parser.add_argument("--ngpu", type=int, default=1, help="number of GPUs to use") + parser.add_argument("--netG", default="", help="path to netG (to continue training)") parser.add_argument("--netD", default="", help="path to netD (to continue training)") parser.add_argument( "--outf", default="./lightning_logs", help="folder to output images and model checkpoints" ) + + +# ------------------------------------------------------------------------------------------------------------ +# Available Automator Flags +# ------------------------------------------------------------------------------------------------------------ +parser.add_argument("--accelerator", type=str, default="ddp", choices=["ddp", "ddp_cpu"]) +parser.add_argument("--gpus", type=int, default=0) +parser.add_argument("--precision", type=int, default=32, choices=[16, 32]) +parser.add_argument("--amp_backend", type=str, default="native", choices=["native"]) + +# required by torch.distributed.launch +# TODO: we need a lightning launcher parser.add_argument("--local_rank", type=int, default=0) opt = parser.parse_args() os.makedirs(opt.outf, exist_ok=True) -ngpu = int(opt.ngpu) nz = 100 @@ -69,7 +81,12 @@ def main(): os.environ["LOCAL_RANK"] = str(opt.local_rank) # os.environ["NODE_RANK"] = str(opt.local_rank) - automator = Automator(gpus=2, accelerator="ddp") + automator = Automator( + accelerator=opt.accelerator, + gpus=opt.gpus, + precision=opt.precision, + amp_backend=opt.amp_backend, + ) dataset = dset.MNIST( root=".", From 420263c25a26f1bf7dee87060f8f7a5b3829894d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 24 Mar 2021 01:18:18 +0100 Subject: [PATCH 019/331] pl example --- pl_examples/automator_examples/gan_example.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index d38c4b5cfd6aa..20343ffc50fc8 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -3,8 +3,7 @@ Launch it with this command: -python -m torch.distributed.launch --nproc_per_node=2 gan_example.py (--accelerator ddp --gpus 2 \ - --precision 16 ) +python -m torch.distributed.launch --nproc_per_node=2 gan_example.py --accelerator ddp --gpus 2 --precision 16 """ from __future__ import print_function From 50812f238ca13b18fbb21331a3ff26246119deb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 24 Mar 2021 01:23:43 +0100 Subject: [PATCH 020/331] update example --- pl_examples/automator_examples/gan_example.py | 10 ++++++---- pl_examples/automator_examples/models.py | 2 ++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index 20343ffc50fc8..7eca7f12a05e8 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -103,7 +103,9 @@ def main(): ) dataloader = automator.setup(dataloader) - # assert isinstance(dataloader.sampler, DistributedSampler) + + if opt.gpus: + assert isinstance(dataloader.sampler, DistributedSampler) netG = Generator() netG.apply(weights_init) @@ -116,8 +118,9 @@ def main(): netG, netD = automator.setup(netG, netD) - # assert isinstance(netG, DistributedDataParallel) - # assert isinstance(netD, DistributedDataParallel) + if opt.gpus: + assert isinstance(netG, DistributedDataParallel) + assert isinstance(netD, DistributedDataParallel) criterion = nn.BCELoss() @@ -143,7 +146,6 @@ def main(): label = torch.full( (batch_size,), real_label, dtype=real_cpu.dtype, device=automator.device ) - output = netD(real_cpu) errD_real = criterion(output, label) automator.backward(errD_real) diff --git a/pl_examples/automator_examples/models.py b/pl_examples/automator_examples/models.py index 70709e30b7b95..a54baf38c1052 100644 --- a/pl_examples/automator_examples/models.py +++ b/pl_examples/automator_examples/models.py @@ -44,6 +44,7 @@ def __init__(self): ) def forward(self, input): + print("autocast enabled in generator: ", torch.is_autocast_enabled()) return self.main(input) @@ -72,5 +73,6 @@ def __init__(self): ) def forward(self, input): + print("autocast enabled in discriminator: ", torch.is_autocast_enabled()) output = self.main(input) return output.view(-1, 1).squeeze(1) \ No newline at end of file From c1ce07e2e491f97c6a16b9e3c1f9a627ff7b291e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 24 Mar 2021 01:40:40 +0100 Subject: [PATCH 021/331] forward context for precision --- pl_examples/automator_examples/gan_example.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index 7eca7f12a05e8..ebe32ea5ec667 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -146,8 +146,11 @@ def main(): label = torch.full( (batch_size,), real_label, dtype=real_cpu.dtype, device=automator.device ) - output = netD(real_cpu) - errD_real = criterion(output, label) + with automator.forward_context(): + # TODO: provide forward context as part of a model wrap + output = netD(real_cpu) + errD_real = criterion(output, label) + automator.backward(errD_real) D_x = output.mean().item() @@ -155,8 +158,12 @@ def main(): noise = torch.randn(batch_size, nz, 1, 1, device=automator.device) fake = netG(noise) label.fill_(fake_label) - output = netD(fake.detach()) - errD_fake = criterion(output, label) + + with automator.forward_context(): + # TODO: provide forward context as part of a model wrap + output = netD(fake.detach()) + errD_fake = criterion(output, label) + automator.backward(errD_fake) D_G_z1 = output.mean().item() errD = errD_real + errD_fake @@ -167,8 +174,10 @@ def main(): ########################### netG.zero_grad() label.fill_(real_label) # fake labels are real for generator cost - output = netD(fake) - errG = criterion(output, label) + with automator.forward_context(): + # TODO: provide forward context as part of a model wrap + output = netD(fake) + errG = criterion(output, label) automator.backward(errG) D_G_z2 = output.mean().item() optimizerG.step() From 75089b1c2a9eadf0f51a1463b7947000166ba585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 24 Mar 2021 01:41:49 +0100 Subject: [PATCH 022/331] fix autocast loss function --- pl_examples/automator_examples/gan_example.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index ebe32ea5ec667..9ddce6cd04222 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -149,8 +149,8 @@ def main(): with automator.forward_context(): # TODO: provide forward context as part of a model wrap output = netD(real_cpu) - errD_real = criterion(output, label) + errD_real = criterion(output, label) automator.backward(errD_real) D_x = output.mean().item() @@ -162,8 +162,8 @@ def main(): with automator.forward_context(): # TODO: provide forward context as part of a model wrap output = netD(fake.detach()) - errD_fake = criterion(output, label) + errD_fake = criterion(output, label) automator.backward(errD_fake) D_G_z1 = output.mean().item() errD = errD_real + errD_fake @@ -177,7 +177,8 @@ def main(): with automator.forward_context(): # TODO: provide forward context as part of a model wrap output = netD(fake) - errG = criterion(output, label) + + errG = criterion(output, label) automator.backward(errG) D_G_z2 = output.mean().item() optimizerG.step() From 572e8f1cb2c34afeea9178b361ae663d781d7cbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 24 Mar 2021 01:44:54 +0100 Subject: [PATCH 023/331] autocast issue --- pl_examples/automator_examples/gan_example.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index 9ddce6cd04222..6d397bc599182 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -150,6 +150,8 @@ def main(): # TODO: provide forward context as part of a model wrap output = netD(real_cpu) + output = output.float() # TODO: Hack, autocast gives us half and criterion complains + errD_real = criterion(output, label) automator.backward(errD_real) D_x = output.mean().item() @@ -163,6 +165,8 @@ def main(): # TODO: provide forward context as part of a model wrap output = netD(fake.detach()) + output = output.float() # TODO: Hack, autocast gives us half and criterion complains + errD_fake = criterion(output, label) automator.backward(errD_fake) D_G_z1 = output.mean().item() @@ -177,7 +181,9 @@ def main(): with automator.forward_context(): # TODO: provide forward context as part of a model wrap output = netD(fake) - + + output = output.float() # TODO: Hack, autocast gives us half and criterion complains + errG = criterion(output, label) automator.backward(errG) D_G_z2 = output.mean().item() From 7a2cd923ad32bb3a9679311e708aeae28c9ce382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 24 Mar 2021 01:46:59 +0100 Subject: [PATCH 024/331] wrap generator --- pl_examples/automator_examples/gan_example.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index 6d397bc599182..45a7c4d2c7ffb 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -158,7 +158,11 @@ def main(): # train with fake noise = torch.randn(batch_size, nz, 1, 1, device=automator.device) - fake = netG(noise) + + with automator.forward_context(): + # TODO: provide forward context as part of a model wrap + fake = netG(noise) + label.fill_(fake_label) with automator.forward_context(): From 98e17d4185c5cc476c1fc966cec66215ce640607 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 24 Mar 2021 02:09:57 +0100 Subject: [PATCH 025/331] test dp and ddp_cpu accelerator options --- pl_examples/automator_examples/gan_example.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index 45a7c4d2c7ffb..7a492de177baa 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -55,8 +55,9 @@ # ------------------------------------------------------------------------------------------------------------ # Available Automator Flags # ------------------------------------------------------------------------------------------------------------ -parser.add_argument("--accelerator", type=str, default="ddp", choices=["ddp", "ddp_cpu"]) +parser.add_argument("--accelerator", type=str, default="ddp", choices=["ddp", "ddp_cpu", "dp"]) parser.add_argument("--gpus", type=int, default=0) +parser.add_argument("--num_processes", type=int, default=1) parser.add_argument("--precision", type=int, default=32, choices=[16, 32]) parser.add_argument("--amp_backend", type=str, default="native", choices=["native"]) @@ -83,6 +84,7 @@ def main(): automator = Automator( accelerator=opt.accelerator, gpus=opt.gpus, + num_processes=opt.num_processes, precision=opt.precision, amp_backend=opt.amp_backend, ) From 77376c074227122a607cc5dd272ab3282c0f58ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 24 Mar 2021 02:12:19 +0100 Subject: [PATCH 026/331] test dataparallel --- pl_examples/automator_examples/gan_example.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index 7a492de177baa..4d27ff241d70b 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -18,6 +18,7 @@ import torchvision.datasets as dset import torchvision.transforms as transforms import torchvision.utils as vutils +from torch.nn import DataParallel from torch.nn.parallel import DistributedDataParallel from torch.utils.data import DistributedSampler @@ -106,7 +107,7 @@ def main(): dataloader = automator.setup(dataloader) - if opt.gpus: + if opt.accelerator == "ddp": assert isinstance(dataloader.sampler, DistributedSampler) netG = Generator() @@ -120,9 +121,12 @@ def main(): netG, netD = automator.setup(netG, netD) - if opt.gpus: + if opt.accelerator == "ddp": assert isinstance(netG, DistributedDataParallel) assert isinstance(netD, DistributedDataParallel) + if opt.accelerator == "dp": + assert isinstance(netD, DataParallel) + assert isinstance(netG, DataParallel) criterion = nn.BCELoss() From 578961f8f0cfd6b83c1b56f01428e35177680ea3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 24 Mar 2021 02:19:17 +0100 Subject: [PATCH 027/331] setup model for DP --- pytorch_lightning/accelerators/accelerator.py | 6 ------ pytorch_lightning/plugins/training_type/dp.py | 7 +++++-- .../plugins/training_type/training_type_plugin.py | 10 ++-------- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/pytorch_lightning/accelerators/accelerator.py b/pytorch_lightning/accelerators/accelerator.py index d9af7e04c5a33..37b06059ec64f 100644 --- a/pytorch_lightning/accelerators/accelerator.py +++ b/pytorch_lightning/accelerators/accelerator.py @@ -139,12 +139,6 @@ def lightning_module(self) -> LightningModule: def root_device(self) -> torch.device: return self.training_type_plugin.root_device - def wrap_model(self, model: nn.Module) -> nn.Module: - return self.training_type_plugin.wrap_model(model) - - def unwrap_model(self, model: nn.Module) -> nn.Module: - return self.training_type_plugin.unwrap_model(model) - def teardown(self) -> None: """This method is called to teardown the training process. It is the right place to release memory and free other ressources. diff --git a/pytorch_lightning/plugins/training_type/dp.py b/pytorch_lightning/plugins/training_type/dp.py index b96b7097d07c7..360eeab0f2309 100644 --- a/pytorch_lightning/plugins/training_type/dp.py +++ b/pytorch_lightning/plugins/training_type/dp.py @@ -14,7 +14,7 @@ from typing import List, Optional import torch -from torch.nn import DataParallel +from torch.nn import DataParallel, Module from pytorch_lightning.core.step_result import Result from pytorch_lightning.overrides.data_parallel import LightningParallelModule @@ -30,7 +30,10 @@ def __init__(self, parallel_devices: Optional[List[torch.device]]): def setup(self, model): # model needs to be moved to the device before it is wrapped model.to(self.root_device) - self._model = DataParallel(LightningParallelModule(model), self.parallel_devices) + self._model = self.setup_model(LightningParallelModule(model)) + + def setup_model(self, model: Module) -> Module: + return DataParallel(model, device_ids=self.parallel_devices) def reduce(self, tensor, *args, **kwargs): """ diff --git a/pytorch_lightning/plugins/training_type/training_type_plugin.py b/pytorch_lightning/plugins/training_type/training_type_plugin.py index 9bdc6390b6873..16b84384058a2 100644 --- a/pytorch_lightning/plugins/training_type/training_type_plugin.py +++ b/pytorch_lightning/plugins/training_type/training_type_plugin.py @@ -57,20 +57,14 @@ def setup_environment(self) -> None: which allows the user to access the accelerator environment before setup is complete. """ - def setup(self, model: 'Module') -> None: + def setup(self, model: Module) -> None: """Called by the accelerator to finish setup.""" def setup_dataloader(self, dataloader: DataLoader) -> DataLoader: """Called by the accelerator. The plugin wraps and modifies the dataloader as needed.""" return dataloader - def setup_model(self, model: nn.Module) -> nn.Module: - return self.wrap_model(model) - - def wrap_model(self, model: nn.Module) -> nn.Module: - return model - - def unwrap_model(self, model: nn.Module) -> nn.Module: + def setup_model(self, model: Module) -> Module: return model @property From 3d80237fc2e4a05398601661c5c06d31d7f67509 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 24 Mar 2021 03:39:18 +0100 Subject: [PATCH 028/331] fix test --- tests/trainer/test_dataloaders.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/trainer/test_dataloaders.py b/tests/trainer/test_dataloaders.py index 52c51777e2a89..530ba0f3948d8 100644 --- a/tests/trainer/test_dataloaders.py +++ b/tests/trainer/test_dataloaders.py @@ -26,6 +26,7 @@ from pytorch_lightning import Callback, Trainer from pytorch_lightning.trainer.states import TrainerState from pytorch_lightning.utilities.data import has_iterable_dataset, has_len +from pytorch_lightning.utilities.distributed import replace_sampler from pytorch_lightning.utilities.exceptions import MisconfigurationException from tests.base import EvalModelTemplate from tests.helpers.boring_model import BoringModel, RandomDataset @@ -1151,11 +1152,5 @@ def test_replace_sampler_with_multiprocessing_context(tmpdir): train = RandomDataset(32, 64) context = 'spawn' train = DataLoader(train, batch_size=32, num_workers=2, multiprocessing_context=context, shuffle=True) - trainer = Trainer( - max_epochs=1, - progress_bar_refresh_rate=20, - overfit_batches=5, - ) - - new_data_loader = trainer.replace_sampler(train, SequentialSampler(train.dataset)) + new_data_loader = replace_sampler(train, SequentialSampler(train.dataset)) assert (new_data_loader.multiprocessing_context == train.multiprocessing_context) From 23ac9572b3b7ff08fc2ca24d19b422f9d58645e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 24 Mar 2021 04:28:08 +0100 Subject: [PATCH 029/331] wrap model instead of context --- pytorch_lightning/accelerators/accelerator.py | 6 ++++ pytorch_lightning/automator/automator.py | 34 ++++++++++++++++--- .../plugins/precision/native_amp.py | 1 - 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/pytorch_lightning/accelerators/accelerator.py b/pytorch_lightning/accelerators/accelerator.py index 37b06059ec64f..60c8ead3323fc 100644 --- a/pytorch_lightning/accelerators/accelerator.py +++ b/pytorch_lightning/accelerators/accelerator.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from contextlib import contextmanager from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, TYPE_CHECKING, Union import torch @@ -447,3 +448,8 @@ def results(self) -> Any: In distributed training, we make sure to transfer the results to the appropriate master process. """ return self.training_type_plugin.results + + @contextmanager + def forward_context(self): + with self.precision_plugin.forward_context(), self.training_type_plugin.forward_context(): + yield diff --git a/pytorch_lightning/automator/automator.py b/pytorch_lightning/automator/automator.py index b3378add03647..9af0e9d6a8150 100644 --- a/pytorch_lightning/automator/automator.py +++ b/pytorch_lightning/automator/automator.py @@ -32,6 +32,23 @@ def step(self, closure=None, **kwargs: Any): return output +class AutomatedModel(nn.Module): + + def __init__(self, module: nn.Module, accelerator: Accelerator): + super().__init__() + self._module = module + self._accelerator = accelerator + + @property + def module(self): + return self._module + + def forward(self, *args, **kwargs): + with self._accelerator.forward_context(): + output = self.module.forward(*args, **kwargs) + return output + + class Automator: def __init__( self, @@ -94,12 +111,20 @@ def setup(self, *objects: Union[nn.Module, Optimizer, DataLoader]): def setup_model(self, *models: nn.Module): # user can call this method independently instead of the general purpose setup method - return [self.training_type_plugin.setup_model(model) for model in models] + models = [ + AutomatedModel(module=self.training_type_plugin.setup_model(model), accelerator=self.accelerator) + for model in models + ] + return models def setup_optimizer(self, *optimizers: Optimizer): # user can call this method independently instead of the general purpose setup method # TODO: let plugin setup optimizer too? - return [AutomatedOptimizer(optimizer=optimizer, accelerator=self.accelerator) for optimizer in optimizers] + optimizers = [ + AutomatedOptimizer(optimizer=optimizer, accelerator=self.accelerator) + for optimizer in optimizers + ] + return optimizers def setup_dataloader(self, *dataloaders: DataLoader): # user can call this method independently instead of the general purpose setup method @@ -110,11 +135,12 @@ def setup_dataloader(self, *dataloaders: DataLoader): return dataloaders def backward(self, tensor: Tensor, *args, **kwargs): + # user will call automator.backward(loss) instead of loss.backward() self.accelerator.run_backward(tensor, *args, **kwargs) @contextmanager def forward_context(self): - with self.precision_plugin.forward_context(), self.training_type_plugin.forward_context(): + with self.accelerator.forward_context(): yield # @contextmanager @@ -135,7 +161,7 @@ def sync(self, data: Any) -> Any: pass def reduce_data(self, data: Any) -> Any: - self.training_type_plugin.reduce(data) + return self.training_type_plugin.reduce(data) def reduce_decision(self, decision: bool) -> bool: return self.training_type_plugin.reduce_boolean_decision(decision) diff --git a/pytorch_lightning/plugins/precision/native_amp.py b/pytorch_lightning/plugins/precision/native_amp.py index c9df56d5675ab..448c55d6f6cfd 100644 --- a/pytorch_lightning/plugins/precision/native_amp.py +++ b/pytorch_lightning/plugins/precision/native_amp.py @@ -74,7 +74,6 @@ def run_backward(self, tensor, *args, **kwargs): super().run_backward(tensor, *args, **kwargs) # self.scaler.unscale_(optimizer) # TODO: needed? - def pre_optimizer_step( self, pl_module: 'LightningModule', From 18e6e63839373c3884b149e5e78059e2a9572779 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 24 Mar 2021 04:33:49 +0100 Subject: [PATCH 030/331] update example assertions --- pl_examples/automator_examples/gan_example.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index 4d27ff241d70b..ac7c8f5b845e1 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -23,7 +23,7 @@ from torch.utils.data import DistributedSampler from pl_examples.automator_examples.models import weights_init, Generator, Discriminator -from pytorch_lightning.automator.automator import Automator +from pytorch_lightning.automator.automator import Automator, AutomatedModel parser = argparse.ArgumentParser() parser.add_argument( @@ -122,11 +122,13 @@ def main(): netG, netD = automator.setup(netG, netD) if opt.accelerator == "ddp": - assert isinstance(netG, DistributedDataParallel) - assert isinstance(netD, DistributedDataParallel) + assert isinstance(netG, AutomatedModel) + assert isinstance(netD, AutomatedModel) + assert isinstance(netG.module, DistributedDataParallel) + assert isinstance(netD.module, DistributedDataParallel) if opt.accelerator == "dp": - assert isinstance(netD, DataParallel) - assert isinstance(netG, DataParallel) + assert isinstance(netD.module, DataParallel) + assert isinstance(netG.module, DataParallel) criterion = nn.BCELoss() From 363b42394373b215ed71f8b325460d92ca70d180 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 24 Mar 2021 04:35:36 +0100 Subject: [PATCH 031/331] update example --- pl_examples/automator_examples/gan_example.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index ac7c8f5b845e1..4c2c9e7203768 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -154,9 +154,9 @@ def main(): label = torch.full( (batch_size,), real_label, dtype=real_cpu.dtype, device=automator.device ) - with automator.forward_context(): - # TODO: provide forward context as part of a model wrap - output = netD(real_cpu) + # with automator.forward_context(): + # TODO: provide forward context as part of a model wrap + output = netD(real_cpu) output = output.float() # TODO: Hack, autocast gives us half and criterion complains @@ -167,15 +167,15 @@ def main(): # train with fake noise = torch.randn(batch_size, nz, 1, 1, device=automator.device) - with automator.forward_context(): - # TODO: provide forward context as part of a model wrap - fake = netG(noise) + # with automator.forward_context(): + # TODO: provide forward context as part of a model wrap + fake = netG(noise) label.fill_(fake_label) - with automator.forward_context(): - # TODO: provide forward context as part of a model wrap - output = netD(fake.detach()) + # with automator.forward_context(): + # TODO: provide forward context as part of a model wrap + output = netD(fake.detach()) output = output.float() # TODO: Hack, autocast gives us half and criterion complains @@ -190,9 +190,9 @@ def main(): ########################### netG.zero_grad() label.fill_(real_label) # fake labels are real for generator cost - with automator.forward_context(): - # TODO: provide forward context as part of a model wrap - output = netD(fake) + # with automator.forward_context(): + # TODO: provide forward context as part of a model wrap + output = netD(fake) output = output.float() # TODO: Hack, autocast gives us half and criterion complains From 0440c3736a34bcae001fe6accbfe0376e46250b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 24 Mar 2021 04:36:57 +0100 Subject: [PATCH 032/331] update example --- pl_examples/automator_examples/gan_example.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index 4c2c9e7203768..1d94a60ea28a4 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -154,11 +154,9 @@ def main(): label = torch.full( (batch_size,), real_label, dtype=real_cpu.dtype, device=automator.device ) - # with automator.forward_context(): - # TODO: provide forward context as part of a model wrap output = netD(real_cpu) - output = output.float() # TODO: Hack, autocast gives us half and criterion complains + # output = output.float() # TODO: Hack, autocast gives us half and criterion complains errD_real = criterion(output, label) automator.backward(errD_real) @@ -166,15 +164,9 @@ def main(): # train with fake noise = torch.randn(batch_size, nz, 1, 1, device=automator.device) - - # with automator.forward_context(): - # TODO: provide forward context as part of a model wrap fake = netG(noise) label.fill_(fake_label) - - # with automator.forward_context(): - # TODO: provide forward context as part of a model wrap output = netD(fake.detach()) output = output.float() # TODO: Hack, autocast gives us half and criterion complains @@ -190,8 +182,6 @@ def main(): ########################### netG.zero_grad() label.fill_(real_label) # fake labels are real for generator cost - # with automator.forward_context(): - # TODO: provide forward context as part of a model wrap output = netD(fake) output = output.float() # TODO: Hack, autocast gives us half and criterion complains From 21d27966bdbde6055c8987649805f598d5d3799a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 24 Mar 2021 04:37:38 +0100 Subject: [PATCH 033/331] update example --- pl_examples/automator_examples/gan_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index 1d94a60ea28a4..1052136070a61 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -156,7 +156,7 @@ def main(): ) output = netD(real_cpu) - # output = output.float() # TODO: Hack, autocast gives us half and criterion complains + output = output.float() # TODO: Hack, autocast gives us half and criterion complains errD_real = criterion(output, label) automator.backward(errD_real) From 7445fc50cc93d3c9607d8eb8abb93c40eb06e1d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 25 Mar 2021 00:38:23 +0100 Subject: [PATCH 034/331] update model+optim setup, distrib setup --- pl_examples/automator_examples/gan_example.py | 31 +++++--- pytorch_lightning/accelerators/accelerator.py | 2 +- pytorch_lightning/automator/automator.py | 71 +++++++++++-------- .../plugins/training_type/ddp.py | 8 +-- pytorch_lightning/plugins/training_type/dp.py | 5 +- .../training_type/training_type_plugin.py | 29 ++++---- 6 files changed, 81 insertions(+), 65 deletions(-) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index 1052136070a61..97f836793626c 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -3,6 +3,7 @@ Launch it with this command: +torchelastic --nproc_per_node=2 python -m torch.distributed.launch --nproc_per_node=2 gan_example.py --accelerator ddp --gpus 2 --precision 16 """ @@ -89,6 +90,11 @@ def main(): precision=opt.precision, amp_backend=opt.amp_backend, ) + # automatorD = AutomatedModel(**kargs) + # automatorG = AutomatedModel(**kargs) + # + # automatorD.setup_optimizer(opt, model1) + dataset = dset.MNIST( root=".", @@ -119,7 +125,10 @@ def main(): automator.to_device(netG) automator.to_device(netD) - netG, netD = automator.setup(netG, netD) + optimizerD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) + optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) + + (netG, netD), (optimizerG, optimizerD) = automator.setup(models=(netG, netD), optimizers=(optimizerG, optimizerD)) if opt.accelerator == "ddp": assert isinstance(netG, AutomatedModel) @@ -136,11 +145,6 @@ def main(): real_label = 1 fake_label = 0 - # setup optimizer - optimizerD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) - optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) - - optimizerG, optimizerG = automator.setup(optimizerG, optimizerD) for epoch in range(opt.niter): for i, data in enumerate(dataloader, 0): @@ -155,11 +159,12 @@ def main(): (batch_size,), real_label, dtype=real_cpu.dtype, device=automator.device ) output = netD(real_cpu) - - output = output.float() # TODO: Hack, autocast gives us half and criterion complains + output = output.float() # required if precision = 16 errD_real = criterion(output, label) + automator.backward(errD_real) + D_x = output.mean().item() # train with fake @@ -169,13 +174,14 @@ def main(): label.fill_(fake_label) output = netD(fake.detach()) - output = output.float() # TODO: Hack, autocast gives us half and criterion complains + output = output.float() # required if precision = 16 errD_fake = criterion(output, label) automator.backward(errD_fake) D_G_z1 = output.mean().item() errD = errD_real + errD_fake - optimizerD.step() + + optimizerD.step() # model inside? ############################ # (2) Update G network: maximize log(D(G(z))) @@ -184,10 +190,13 @@ def main(): label.fill_(real_label) # fake labels are real for generator cost output = netD(fake) - output = output.float() # TODO: Hack, autocast gives us half and criterion complains + output = output.float() # required if precision = 16 errG = criterion(output, label) + + # document automator.backward(errG) + D_G_z2 = output.mean().item() optimizerG.step() diff --git a/pytorch_lightning/accelerators/accelerator.py b/pytorch_lightning/accelerators/accelerator.py index 60c8ead3323fc..918e5d4d62fee 100644 --- a/pytorch_lightning/accelerators/accelerator.py +++ b/pytorch_lightning/accelerators/accelerator.py @@ -95,7 +95,7 @@ def setup_dataloader(self, dataloader: DataLoader) -> DataLoader: return self.training_type_plugin.setup_dataloader(dataloader) def setup_model(self, model: nn.Module) -> nn.Module: - return self.training_type_plugin.setup_model(model) + return self.training_type_plugin.setup_model_and_optimizers(model, None) def start_training(self, trainer: 'Trainer') -> None: self.training_type_plugin.start_training(trainer) diff --git a/pytorch_lightning/automator/automator.py b/pytorch_lightning/automator/automator.py index 9af0e9d6a8150..5bfb865c94f63 100644 --- a/pytorch_lightning/automator/automator.py +++ b/pytorch_lightning/automator/automator.py @@ -1,12 +1,13 @@ from weakref import proxy from collections import Callable from contextlib import contextmanager -from typing import Any, Union, Optional +from typing import Any, Union, Optional, Sequence, Tuple import torch.nn as nn from torch import Tensor from torch.optim import Optimizer from torch.utils.data import DataLoader +import torch.multiprocessing as mp from pytorch_lightning.accelerators import Accelerator from pytorch_lightning.trainer.connectors.accelerator_connector import ( @@ -22,7 +23,6 @@ def __init__(self, optimizer: Optimizer, accelerator: Accelerator): self._accelerator = accelerator def step(self, closure=None, **kwargs: Any): - # TODO: do precision magic here print("running automated step") output = self._accelerator.run_optimizer_step( self.optimizer, @@ -62,6 +62,11 @@ def __init__( amp_backend: str = "native", amp_level: str = "O2", ): + + if accelerator == "ddp_spawn": + raise + + backend_connector = AcceleratorConnector( gpus=gpus, tpu_cores=tpus, @@ -81,6 +86,10 @@ def __init__( ) self.accelerator = backend_connector.select_accelerator() + # TODO: Do we need to initialize distributed at the very beginning + # any reason to delay?? + self.accelerator.setup_environment() + @property def training_type_plugin(self): return self.accelerator.training_type_plugin @@ -94,37 +103,33 @@ def device(self): # the device on the local rank return self.training_type_plugin.root_device - def setup(self, *objects: Union[nn.Module, Optimizer, DataLoader]): + def setup( + self, + models: Union[nn.Module, Sequence[nn.Module]], + optimizers: Union[Optimizer, Sequence[Optimizer]], + ): # wrap all objects passed in and return them in the same order - wrapped_objects = [] - for obj in objects: - if isinstance(obj, nn.Module): - wrapped_objects.extend(self.setup_model(obj)) - if isinstance(obj, Optimizer): - wrapped_objects.extend(self.setup_optimizer(obj)) - if isinstance(obj, DataLoader): - wrapped_objects.extend(self.setup_dataloader(obj)) - - if len(wrapped_objects) == 1: - return wrapped_objects[0] - return wrapped_objects - - def setup_model(self, *models: nn.Module): - # user can call this method independently instead of the general purpose setup method + models = [models] if len(models) == 1 else models + optimizers = [optimizers] if len(optimizers) == 1 else optimizers + models, optimizers = self._setup_models_and_optimizers(models, optimizers) + + models = models[0] if len(models) == 1 else models + optimizers = optimizers[0] if len(optimizers) == 1 else optimizers + return models, optimizers + + def _setup_models_and_optimizers(self, models: Sequence[nn.Module], optimizers: Sequence[Optimizer]): + # Let accelerator/plugin wrap and connect the models and optimizers + models, optimizers = self.training_type_plugin.setup_models_and_optimizers(models, optimizers) models = [ - AutomatedModel(module=self.training_type_plugin.setup_model(model), accelerator=self.accelerator) + AutomatedModel(module=model, accelerator=self.accelerator) for model in models ] - return models - - def setup_optimizer(self, *optimizers: Optimizer): - # user can call this method independently instead of the general purpose setup method - # TODO: let plugin setup optimizer too? optimizers = [ - AutomatedOptimizer(optimizer=optimizer, accelerator=self.accelerator) + AutomatedOptimizer( + optimizer=optimizer, accelerator=self.accelerator) for optimizer in optimizers ] - return optimizers + return models, optimizers def setup_dataloader(self, *dataloaders: DataLoader): # user can call this method independently instead of the general purpose setup method @@ -148,9 +153,12 @@ def forward_context(self): # yield # # @contextmanager - # def optimizer_step_context(self, *args, **kwargs): - # # necessary for deepspeed + scaling - # yield + def optimizer_step_context(self, model=None, optimizer=None): + # necessary for deepspeed + scaling + temp = optimizer.step + optimizer.step = model.step + yield + optimizer.step = temp def to_device(self, obj: Union[nn.Module, Tensor]) -> Union[nn.Module, Tensor]: if isinstance(obj, nn.Module): @@ -158,6 +166,7 @@ def to_device(self, obj: Union[nn.Module, Tensor]) -> Union[nn.Module, Tensor]: return move_data_to_device(obj, device=self.device) def sync(self, data: Any) -> Any: + # all_gather pass def reduce_data(self, data: Any) -> Any: @@ -175,3 +184,7 @@ def save_checkpoint(self, filepath): def execute_on_rank(self, func: Callable, rank: int): pass + + def spawn(self, function: Callable, *args: Any): + # ctx = mp.spawn(function, args, nprocs=..., ...) + pass diff --git a/pytorch_lightning/plugins/training_type/ddp.py b/pytorch_lightning/plugins/training_type/ddp.py index 0e408d942a19d..a73c367fc4836 100644 --- a/pytorch_lightning/plugins/training_type/ddp.py +++ b/pytorch_lightning/plugins/training_type/ddp.py @@ -92,17 +92,15 @@ def setup_environment(self): self.setup_distributed() - def setup_model(self, model: nn.Module) -> DistributedDataParallel: - self.setup_distributed() # setup distributed if it is not already initialized + def setup_model(self, model: nn.Module) -> nn.Module: model = DistributedDataParallel( - model.to(self.root_device), + module=model.to(self.root_device), device_ids=self.determine_ddp_device_ids(), **self._ddp_kwargs, ) return model def setup_dataloader(self, dataloader: DataLoader) -> DataLoader: - self.setup_distributed() # setup distributed if it is not already initialized kwargs = self.distributed_sampler_kwargs sampler = DistributedSampler(dataloader.dataset, **kwargs) dataloader = replace_sampler(dataloader, sampler) @@ -233,7 +231,7 @@ def pre_configure_ddp(self): def configure_ddp(self): self.pre_configure_ddp() - self._model = self.setup_model(LightningDistributedModule(self.model)) + self._model = self.setup_model_and_optimizers(LightningDistributedModule(self.model), None) def determine_ddp_device_ids(self): if self.root_device.type == "cpu": diff --git a/pytorch_lightning/plugins/training_type/dp.py b/pytorch_lightning/plugins/training_type/dp.py index 360eeab0f2309..ac1710dd776bc 100644 --- a/pytorch_lightning/plugins/training_type/dp.py +++ b/pytorch_lightning/plugins/training_type/dp.py @@ -11,10 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Optional +from typing import List, Optional, Sequence import torch from torch.nn import DataParallel, Module +from torch.optim import Optimizer from pytorch_lightning.core.step_result import Result from pytorch_lightning.overrides.data_parallel import LightningParallelModule @@ -33,7 +34,7 @@ def setup(self, model): self._model = self.setup_model(LightningParallelModule(model)) def setup_model(self, model: Module) -> Module: - return DataParallel(model, device_ids=self.parallel_devices) + return DataParallel(module=model, device_ids=self.parallel_devices) def reduce(self, tensor, *args, **kwargs): """ diff --git a/pytorch_lightning/plugins/training_type/training_type_plugin.py b/pytorch_lightning/plugins/training_type/training_type_plugin.py index 16b84384058a2..9ae58282fd7f1 100644 --- a/pytorch_lightning/plugins/training_type/training_type_plugin.py +++ b/pytorch_lightning/plugins/training_type/training_type_plugin.py @@ -12,15 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, Iterable, Optional, TYPE_CHECKING, Union +from typing import Any, Callable, Dict, Iterable, Optional, TYPE_CHECKING, Union, Sequence import torch -import torch.nn as nn from torch.nn import Module from torch.optim import Optimizer from torch.utils.data import DataLoader -from pytorch_lightning.core.lightning import LightningModule from pytorch_lightning.overrides.base import unwrap_lightning_module from pytorch_lightning.plugins.base_plugin import Plugin @@ -28,17 +26,6 @@ from pytorch_lightning.trainer.trainer import Trainer -def call_hook(model: nn.Module, name: str, *args, **kwargs) -> Any: - """ - Call a hook on the model if it is available. - - Return: - None if hook is undefined, and the result of the hook otherwise. - """ - if hasattr(model, name) and callable(model.name): - return getattr(model, name)(*args, **kwargs) - - class TrainingTypePlugin(Plugin, ABC): """A Plugin to change the behaviour of the training, validation and test-loop.""" @@ -46,7 +33,7 @@ def __init__(self) -> None: self._model = None self._results = None - def connect(self, model: 'Module') -> None: + def connect(self, model: Module) -> None: """Called by the accelerator to connect the accelerator and the model with this plugin""" self.model = model @@ -64,9 +51,17 @@ def setup_dataloader(self, dataloader: DataLoader) -> DataLoader: """Called by the accelerator. The plugin wraps and modifies the dataloader as needed.""" return dataloader + def setup_models_and_optimizers(self, models: Sequence[Module], optimizers: Sequence[Optimizer]): + models = [self.setup_model(model) for model in models] + optimizers = [self.setup_optimizer(optimizer) for optimizer in optimizers] + return models, optimizers + def setup_model(self, model: Module) -> Module: return model + def setup_optimizer(self, optimizer: Optimizer) -> Optimizer: + return optimizer + @property @abstractmethod def on_gpu(self) -> bool: @@ -132,7 +127,7 @@ def model(self, new_model: Module) -> None: self._model = new_model @property - def lightning_module(self) -> LightningModule: + def lightning_module(self) -> Module: """Returns the pure LightningModule without potential wrappers""" return unwrap_lightning_module(self._model) @@ -196,7 +191,7 @@ def process_dataloader(self, dataloader: Union[Iterable, DataLoader]) -> Union[I """ return dataloader - def init_optimizers(self, trainer: "Trainer", model: LightningModule): + def init_optimizers(self, trainer: "Trainer", model: Module): return trainer.init_optimizers(model) def optimizer_step(self, optimizer: torch.optim.Optimizer, lambda_closure: Callable, **kwargs): From f4c917eaf6c0efbccebc99c097634d8a8b8de53f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 25 Mar 2021 00:41:24 +0100 Subject: [PATCH 035/331] set env variable to prevent spawning children --- pl_examples/automator_examples/gan_example.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index 97f836793626c..00eee73f90ce6 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -82,6 +82,7 @@ def main(): # TODO: how do we handle this? os.environ["LOCAL_RANK"] = str(opt.local_rank) # os.environ["NODE_RANK"] = str(opt.local_rank) + os.environ["PL_IN_DDP_SUBPROCESS"] = "1" automator = Automator( accelerator=opt.accelerator, From cad9bf00fdcdef88256c2c5750b466aad9a2def2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 25 Mar 2021 00:43:41 +0100 Subject: [PATCH 036/331] setup dataloader --- pl_examples/automator_examples/gan_example.py | 4 +--- pytorch_lightning/automator/automator.py | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index 00eee73f90ce6..697b601be955e 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -95,8 +95,6 @@ def main(): # automatorG = AutomatedModel(**kargs) # # automatorD.setup_optimizer(opt, model1) - - dataset = dset.MNIST( root=".", download=True, @@ -112,7 +110,7 @@ def main(): dataset, batch_size=opt.batchSize, shuffle=True, num_workers=opt.workers ) - dataloader = automator.setup(dataloader) + dataloader = automator.setup_dataloader(dataloader) if opt.accelerator == "ddp": assert isinstance(dataloader.sampler, DistributedSampler) diff --git a/pytorch_lightning/automator/automator.py b/pytorch_lightning/automator/automator.py index 5bfb865c94f63..27b1ffc7d6da1 100644 --- a/pytorch_lightning/automator/automator.py +++ b/pytorch_lightning/automator/automator.py @@ -137,6 +137,7 @@ def setup_dataloader(self, *dataloaders: DataLoader): self.training_type_plugin.setup_dataloader(dataloader) for dataloader in dataloaders ] + dataloaders = dataloaders[0] if len(dataloaders) == 1 else dataloaders return dataloaders def backward(self, tensor: Tensor, *args, **kwargs): From 4e91f77c8e4cf6bafc5f2c674e5968d2a252f5c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 9 Apr 2021 10:57:36 +0200 Subject: [PATCH 037/331] set ddp flag default --- pl_examples/automator_examples/gan_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index 697b601be955e..77e5d536d54dc 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -57,7 +57,7 @@ # ------------------------------------------------------------------------------------------------------------ # Available Automator Flags # ------------------------------------------------------------------------------------------------------------ -parser.add_argument("--accelerator", type=str, default="ddp", choices=["ddp", "ddp_cpu", "dp"]) +parser.add_argument("--accelerator", type=str, default=None, choices=["ddp", "ddp_cpu", "dp"]) parser.add_argument("--gpus", type=int, default=0) parser.add_argument("--num_processes", type=int, default=1) parser.add_argument("--precision", type=int, default=32, choices=[16, 32]) From d5508c75300394e399caad7e32712e1e16e420c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 12 Oct 2021 18:09:00 +0200 Subject: [PATCH 038/331] rename --- pl_examples/automator_examples/gan_example.py | 43 ++++++------------- pytorch_lightning/automator/__init__.py | 0 pytorch_lightning/lite/__init__.py | 17 ++++++++ .../{automator/automator.py => lite/lite.py} | 24 +++-------- pytorch_lightning/trainer/data_loading.py | 1 + 5 files changed, 37 insertions(+), 48 deletions(-) delete mode 100644 pytorch_lightning/automator/__init__.py create mode 100644 pytorch_lightning/lite/__init__.py rename pytorch_lightning/{automator/automator.py => lite/lite.py} (91%) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py index 77e5d536d54dc..01a53d1437924 100644 --- a/pl_examples/automator_examples/gan_example.py +++ b/pl_examples/automator_examples/gan_example.py @@ -24,12 +24,10 @@ from torch.utils.data import DistributedSampler from pl_examples.automator_examples.models import weights_init, Generator, Discriminator -from pytorch_lightning.automator.automator import Automator, AutomatedModel +from pytorch_lightning.lite.automator import LightningLite, AutomatedModel parser = argparse.ArgumentParser() -parser.add_argument( - "--workers", type=int, help="number of data loading workers", default=0 -) +parser.add_argument("--workers", type=int, help="number of data loading workers", default=0) parser.add_argument("--batchSize", type=int, default=64, help="input batch size") parser.add_argument( "--imageSize", @@ -37,25 +35,17 @@ default=64, help="the height / width of the input image to network", ) -parser.add_argument( - "--niter", type=int, default=25, help="number of epochs to train for" -) -parser.add_argument( - "--lr", type=float, default=0.0002, help="learning rate, default=0.0002" -) -parser.add_argument( - "--beta1", type=float, default=0.5, help="beta1 for adam. default=0.5" -) +parser.add_argument("--niter", type=int, default=25, help="number of epochs to train for") +parser.add_argument("--lr", type=float, default=0.0002, help="learning rate, default=0.0002") +parser.add_argument("--beta1", type=float, default=0.5, help="beta1 for adam. default=0.5") parser.add_argument("--netG", default="", help="path to netG (to continue training)") parser.add_argument("--netD", default="", help="path to netD (to continue training)") -parser.add_argument( - "--outf", default="./lightning_logs", help="folder to output images and model checkpoints" -) +parser.add_argument("--outf", default="./lightning_logs", help="folder to output images and model checkpoints") # ------------------------------------------------------------------------------------------------------------ -# Available Automator Flags +# Available LightningLite Flags # ------------------------------------------------------------------------------------------------------------ parser.add_argument("--accelerator", type=str, default=None, choices=["ddp", "ddp_cpu", "dp"]) parser.add_argument("--gpus", type=int, default=0) @@ -84,15 +74,15 @@ def main(): # os.environ["NODE_RANK"] = str(opt.local_rank) os.environ["PL_IN_DDP_SUBPROCESS"] = "1" - automator = Automator( + automator = LightningLite( accelerator=opt.accelerator, gpus=opt.gpus, num_processes=opt.num_processes, precision=opt.precision, amp_backend=opt.amp_backend, ) - # automatorD = AutomatedModel(**kargs) - # automatorG = AutomatedModel(**kargs) + # automatorD = LiteModel(**kargs) + # automatorG = LiteModel(**kargs) # # automatorD.setup_optimizer(opt, model1) dataset = dset.MNIST( @@ -106,9 +96,7 @@ def main(): ] ), ) - dataloader = torch.utils.data.DataLoader( - dataset, batch_size=opt.batchSize, shuffle=True, num_workers=opt.workers - ) + dataloader = torch.utils.data.DataLoader(dataset, batch_size=opt.batchSize, shuffle=True, num_workers=opt.workers) dataloader = automator.setup_dataloader(dataloader) @@ -144,7 +132,6 @@ def main(): real_label = 1 fake_label = 0 - for epoch in range(opt.niter): for i, data in enumerate(dataloader, 0): ############################ @@ -154,9 +141,7 @@ def main(): netD.zero_grad() real_cpu = automator.to_device(data[0]) batch_size = real_cpu.size(0) - label = torch.full( - (batch_size,), real_label, dtype=real_cpu.dtype, device=automator.device - ) + label = torch.full((batch_size,), real_label, dtype=real_cpu.dtype, device=automator.device) output = netD(real_cpu) output = output.float() # required if precision = 16 @@ -214,9 +199,7 @@ def main(): ) ) if i % 100 == 0: - vutils.save_image( - real_cpu, "%s/real_samples.png" % opt.outf, normalize=True - ) + vutils.save_image(real_cpu, "%s/real_samples.png" % opt.outf, normalize=True) fake = netG(fixed_noise) vutils.save_image( fake.detach(), diff --git a/pytorch_lightning/automator/__init__.py b/pytorch_lightning/automator/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/pytorch_lightning/lite/__init__.py b/pytorch_lightning/lite/__init__.py new file mode 100644 index 0000000000000..f4634fe54e548 --- /dev/null +++ b/pytorch_lightning/lite/__init__.py @@ -0,0 +1,17 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pytorch_lightning.lite.lite import LightningLite + +__all__ = ["LightningLite"] diff --git a/pytorch_lightning/automator/automator.py b/pytorch_lightning/lite/lite.py similarity index 91% rename from pytorch_lightning/automator/automator.py rename to pytorch_lightning/lite/lite.py index 27b1ffc7d6da1..9fec3e21c0f0b 100644 --- a/pytorch_lightning/automator/automator.py +++ b/pytorch_lightning/lite/lite.py @@ -16,7 +16,7 @@ from pytorch_lightning.utilities import move_data_to_device -class AutomatedOptimizer(Optimizer): +class LiteOptimizer(Optimizer): def __init__(self, optimizer: Optimizer, accelerator: Accelerator): super().__init__(params=optimizer.param_groups, defaults={}) self.optimizer = optimizer @@ -32,8 +32,7 @@ def step(self, closure=None, **kwargs: Any): return output -class AutomatedModel(nn.Module): - +class LiteModel(nn.Module): def __init__(self, module: nn.Module, accelerator: Accelerator): super().__init__() self._module = module @@ -49,7 +48,7 @@ def forward(self, *args, **kwargs): return output -class Automator: +class LightningLite: def __init__( self, accelerator=None, @@ -66,7 +65,6 @@ def __init__( if accelerator == "ddp_spawn": raise - backend_connector = AcceleratorConnector( gpus=gpus, tpu_cores=tpus, @@ -120,23 +118,13 @@ def setup( def _setup_models_and_optimizers(self, models: Sequence[nn.Module], optimizers: Sequence[Optimizer]): # Let accelerator/plugin wrap and connect the models and optimizers models, optimizers = self.training_type_plugin.setup_models_and_optimizers(models, optimizers) - models = [ - AutomatedModel(module=model, accelerator=self.accelerator) - for model in models - ] - optimizers = [ - AutomatedOptimizer( - optimizer=optimizer, accelerator=self.accelerator) - for optimizer in optimizers - ] + models = [LiteModel(module=model, accelerator=self.accelerator) for model in models] + optimizers = [LiteOptimizer(optimizer=optimizer, accelerator=self.accelerator) for optimizer in optimizers] return models, optimizers def setup_dataloader(self, *dataloaders: DataLoader): # user can call this method independently instead of the general purpose setup method - dataloaders = [ - self.training_type_plugin.setup_dataloader(dataloader) - for dataloader in dataloaders - ] + dataloaders = [self.training_type_plugin.setup_dataloader(dataloader) for dataloader in dataloaders] dataloaders = dataloaders[0] if len(dataloaders) == 1 else dataloaders return dataloaders diff --git a/pytorch_lightning/trainer/data_loading.py b/pytorch_lightning/trainer/data_loading.py index 4e7a093b70b8a..377a9cca3c4cc 100644 --- a/pytorch_lightning/trainer/data_loading.py +++ b/pytorch_lightning/trainer/data_loading.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import inspect import multiprocessing import os from abc import ABC From 5ffc6dfb111a0669763bc20601a477f0a213ec9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 12 Oct 2021 18:10:14 +0200 Subject: [PATCH 039/331] acc --- pytorch_lightning/lite/lite.py | 35 ++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 9fec3e21c0f0b..d12f18d7bf11b 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -9,6 +9,7 @@ from torch.utils.data import DataLoader import torch.multiprocessing as mp +from pytorch_lightning import Trainer from pytorch_lightning.accelerators import Accelerator from pytorch_lightning.trainer.connectors.accelerator_connector import ( AcceleratorConnector, @@ -52,35 +53,37 @@ class LightningLite: def __init__( self, accelerator=None, - plugin=None, + plugins=None, gpus=None, - tpus=None, + tpu_cores=None, + ipus=None, num_processes=1, + devices=None, num_nodes=1, precision=32, amp_backend: str = "native", amp_level: str = "O2", + replace_sampler_ddp=True, ): - - if accelerator == "ddp_spawn": - raise - + gpu_ids, tpu_cores = Trainer._parse_devices(gpus=gpus, auto_select_gpus=False, tpu_cores=tpu_cores) backend_connector = AcceleratorConnector( - gpus=gpus, - tpu_cores=tpus, num_processes=num_processes, - distributed_backend=accelerator, + devices=devices, + tpu_cores=tpu_cores, + ipus=ipus, + distributed_backend=None, # TODO: remove + accelerator=accelerator, + gpus=gpus, + gpu_ids=gpu_ids, num_nodes=num_nodes, + sync_batchnorm=False, # TODO: add support? + benchmark=False, + replace_sampler_ddp=replace_sampler_ddp, + deterministic=False, precision=precision, amp_type=amp_backend, amp_level=amp_level, - plugins=plugin, - # TODO: - deterministic=False, - sync_batchnorm=False, - benchmark=False, - replace_sampler_ddp=True, - auto_select_gpus=False, + plugins=plugins, ) self.accelerator = backend_connector.select_accelerator() From 92934a527da2a015384cf8246d594af01dd86d0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 12 Oct 2021 18:15:17 +0200 Subject: [PATCH 040/331] fixes --- pl_examples/automator_examples/gan_example.py | 215 ------------------ .../__init__.py | 0 pl_examples/lite_examples/gan_example.py | 172 ++++++++++++++ .../models.py | 0 pytorch_lightning/accelerators/accelerator.py | 12 +- .../plugins/precision/native_amp.py | 9 +- .../plugins/training_type/ddp.py | 6 +- .../plugins/training_type/tpu_spawn.py | 5 +- .../training_type/training_type_plugin.py | 3 +- .../connectors/accelerator_connector.py | 4 +- 10 files changed, 193 insertions(+), 233 deletions(-) delete mode 100644 pl_examples/automator_examples/gan_example.py rename pl_examples/{automator_examples => lite_examples}/__init__.py (100%) create mode 100644 pl_examples/lite_examples/gan_example.py rename pl_examples/{automator_examples => lite_examples}/models.py (100%) diff --git a/pl_examples/automator_examples/gan_example.py b/pl_examples/automator_examples/gan_example.py deleted file mode 100644 index 01a53d1437924..0000000000000 --- a/pl_examples/automator_examples/gan_example.py +++ /dev/null @@ -1,215 +0,0 @@ -""" -DCGAN - Adapted from pytorch/examples - -Launch it with this command: - -torchelastic --nproc_per_node=2 -python -m torch.distributed.launch --nproc_per_node=2 gan_example.py --accelerator ddp --gpus 2 --precision 16 - -""" -from __future__ import print_function -import argparse -import os -import random -import torch -import torch.nn as nn -import torch.nn.parallel -import torch.optim as optim -import torch.utils.data -import torchvision.datasets as dset -import torchvision.transforms as transforms -import torchvision.utils as vutils -from torch.nn import DataParallel -from torch.nn.parallel import DistributedDataParallel -from torch.utils.data import DistributedSampler - -from pl_examples.automator_examples.models import weights_init, Generator, Discriminator -from pytorch_lightning.lite.automator import LightningLite, AutomatedModel - -parser = argparse.ArgumentParser() -parser.add_argument("--workers", type=int, help="number of data loading workers", default=0) -parser.add_argument("--batchSize", type=int, default=64, help="input batch size") -parser.add_argument( - "--imageSize", - type=int, - default=64, - help="the height / width of the input image to network", -) -parser.add_argument("--niter", type=int, default=25, help="number of epochs to train for") -parser.add_argument("--lr", type=float, default=0.0002, help="learning rate, default=0.0002") -parser.add_argument("--beta1", type=float, default=0.5, help="beta1 for adam. default=0.5") - -parser.add_argument("--netG", default="", help="path to netG (to continue training)") -parser.add_argument("--netD", default="", help="path to netD (to continue training)") -parser.add_argument("--outf", default="./lightning_logs", help="folder to output images and model checkpoints") - - -# ------------------------------------------------------------------------------------------------------------ -# Available LightningLite Flags -# ------------------------------------------------------------------------------------------------------------ -parser.add_argument("--accelerator", type=str, default=None, choices=["ddp", "ddp_cpu", "dp"]) -parser.add_argument("--gpus", type=int, default=0) -parser.add_argument("--num_processes", type=int, default=1) -parser.add_argument("--precision", type=int, default=32, choices=[16, 32]) -parser.add_argument("--amp_backend", type=str, default="native", choices=["native"]) - -# required by torch.distributed.launch -# TODO: we need a lightning launcher -parser.add_argument("--local_rank", type=int, default=0) - -opt = parser.parse_args() -os.makedirs(opt.outf, exist_ok=True) - -nz = 100 - - -def main(): - random.seed(123) - torch.manual_seed(123) - - # TODO: how do we handle this in Accelerator? - # torch.cuda.set_device(opt.local_rank) - # TODO: how do we handle this? - os.environ["LOCAL_RANK"] = str(opt.local_rank) - # os.environ["NODE_RANK"] = str(opt.local_rank) - os.environ["PL_IN_DDP_SUBPROCESS"] = "1" - - automator = LightningLite( - accelerator=opt.accelerator, - gpus=opt.gpus, - num_processes=opt.num_processes, - precision=opt.precision, - amp_backend=opt.amp_backend, - ) - # automatorD = LiteModel(**kargs) - # automatorG = LiteModel(**kargs) - # - # automatorD.setup_optimizer(opt, model1) - dataset = dset.MNIST( - root=".", - download=True, - transform=transforms.Compose( - [ - transforms.Resize(opt.imageSize), - transforms.ToTensor(), - transforms.Normalize((0.5,), (0.5,)), - ] - ), - ) - dataloader = torch.utils.data.DataLoader(dataset, batch_size=opt.batchSize, shuffle=True, num_workers=opt.workers) - - dataloader = automator.setup_dataloader(dataloader) - - if opt.accelerator == "ddp": - assert isinstance(dataloader.sampler, DistributedSampler) - - netG = Generator() - netG.apply(weights_init) - - netD = Discriminator() - netD.apply(weights_init) - - automator.to_device(netG) - automator.to_device(netD) - - optimizerD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) - optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) - - (netG, netD), (optimizerG, optimizerD) = automator.setup(models=(netG, netD), optimizers=(optimizerG, optimizerD)) - - if opt.accelerator == "ddp": - assert isinstance(netG, AutomatedModel) - assert isinstance(netD, AutomatedModel) - assert isinstance(netG.module, DistributedDataParallel) - assert isinstance(netD.module, DistributedDataParallel) - if opt.accelerator == "dp": - assert isinstance(netD.module, DataParallel) - assert isinstance(netG.module, DataParallel) - - criterion = nn.BCELoss() - - fixed_noise = torch.randn(opt.batchSize, nz, 1, 1, device=automator.device) - real_label = 1 - fake_label = 0 - - for epoch in range(opt.niter): - for i, data in enumerate(dataloader, 0): - ############################ - # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z))) - ########################### - # train with real - netD.zero_grad() - real_cpu = automator.to_device(data[0]) - batch_size = real_cpu.size(0) - label = torch.full((batch_size,), real_label, dtype=real_cpu.dtype, device=automator.device) - output = netD(real_cpu) - output = output.float() # required if precision = 16 - - errD_real = criterion(output, label) - - automator.backward(errD_real) - - D_x = output.mean().item() - - # train with fake - noise = torch.randn(batch_size, nz, 1, 1, device=automator.device) - fake = netG(noise) - - label.fill_(fake_label) - output = netD(fake.detach()) - - output = output.float() # required if precision = 16 - - errD_fake = criterion(output, label) - automator.backward(errD_fake) - D_G_z1 = output.mean().item() - errD = errD_real + errD_fake - - optimizerD.step() # model inside? - - ############################ - # (2) Update G network: maximize log(D(G(z))) - ########################### - netG.zero_grad() - label.fill_(real_label) # fake labels are real for generator cost - output = netD(fake) - - output = output.float() # required if precision = 16 - - errG = criterion(output, label) - - # document - automator.backward(errG) - - D_G_z2 = output.mean().item() - optimizerG.step() - - print( - "[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)): %.4f / %.4f" - % ( - epoch, - opt.niter, - i, - len(dataloader), - errD.item(), - errG.item(), - D_x, - D_G_z1, - D_G_z2, - ) - ) - if i % 100 == 0: - vutils.save_image(real_cpu, "%s/real_samples.png" % opt.outf, normalize=True) - fake = netG(fixed_noise) - vutils.save_image( - fake.detach(), - "%s/fake_samples_epoch_%03d.png" % (opt.outf, epoch), - normalize=True, - ) - # do checkpointing - torch.save(netG.state_dict(), "%s/netG_epoch_%d.pth" % (opt.outf, epoch)) - torch.save(netD.state_dict(), "%s/netD_epoch_%d.pth" % (opt.outf, epoch)) - - -if __name__ == "__main__": - main() diff --git a/pl_examples/automator_examples/__init__.py b/pl_examples/lite_examples/__init__.py similarity index 100% rename from pl_examples/automator_examples/__init__.py rename to pl_examples/lite_examples/__init__.py diff --git a/pl_examples/lite_examples/gan_example.py b/pl_examples/lite_examples/gan_example.py new file mode 100644 index 0000000000000..197bd2e19f008 --- /dev/null +++ b/pl_examples/lite_examples/gan_example.py @@ -0,0 +1,172 @@ +""" +DCGAN - Adapted from pytorch/examples + +Launch it with this command: + +python -m torch.distributed.launch --nproc_per_node=2 gan_example.py + +""" +from __future__ import print_function +import argparse +import os +import random +import torch +import torch.nn as nn +import torch.nn.parallel +import torch.optim as optim +import torch.utils.data +import torchvision.datasets as dset +import torchvision.transforms as transforms +import torchvision.utils as vutils + +from pl_examples.lite_examples.models import weights_init, Generator, Discriminator +from pytorch_lightning.lite import LightningLite + +parser = argparse.ArgumentParser() +parser.add_argument("--workers", type=int, help="number of data loading workers", default=0) +parser.add_argument("--batchSize", type=int, default=64, help="input batch size") +parser.add_argument( + "--imageSize", + type=int, + default=64, + help="the height / width of the input image to network", +) +parser.add_argument("--niter", type=int, default=25, help="number of epochs to train for") +parser.add_argument("--lr", type=float, default=0.0002, help="learning rate, default=0.0002") +parser.add_argument("--beta1", type=float, default=0.5, help="beta1 for adam. default=0.5") +parser.add_argument("--ngpu", type=int, default=1, help="number of GPUs to use") +parser.add_argument("--netG", default="", help="path to netG (to continue training)") +parser.add_argument("--netD", default="", help="path to netD (to continue training)") +parser.add_argument("--outf", default="./lightning_logs", help="folder to output images and model checkpoints") +parser.add_argument("--local_rank", type=int, default=0) + +opt = parser.parse_args() +os.makedirs(opt.outf, exist_ok=True) +ngpu = int(opt.ngpu) + +nz = 100 + + +class Lite(LightningLite): + def run(self): + random.seed(123) + torch.manual_seed(123) + + # TODO: how do we handle this in Accelerator? + # torch.cuda.set_device(opt.local_rank) + # TODO: how do we handle this? + os.environ["LOCAL_RANK"] = str(opt.local_rank) + # os.environ["NODE_RANK"] = str(opt.local_rank) + + dataset = dset.MNIST( + root=".", + download=True, + transform=transforms.Compose( + [ + transforms.Resize(opt.imageSize), + transforms.ToTensor(), + transforms.Normalize((0.5,), (0.5,)), + ] + ), + ) + dataloader = torch.utils.data.DataLoader( + dataset, batch_size=opt.batchSize, shuffle=True, num_workers=opt.workers + ) + + dataloader = lite.setup(dataloader) + # assert isinstance(dataloader.sampler, DistributedSampler) + + netG = Generator() + netG.apply(weights_init) + + netD = Discriminator() + netD.apply(weights_init) + + lite.to_device(netG) + lite.to_device(netD) + + netG, netD = lite.setup(netG, netD) + + # assert isinstance(netG, DistributedDataParallel) + # assert isinstance(netD, DistributedDataParallel) + + criterion = nn.BCELoss() + + fixed_noise = torch.randn(opt.batchSize, nz, 1, 1, device=lite.device) + real_label = 1 + fake_label = 0 + + # setup optimizer + optimizerD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) + optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) + + optimizerG, optimizerG = lite.setup(optimizerG, optimizerD) + + for epoch in range(opt.niter): + for i, data in enumerate(dataloader, 0): + ############################ + # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z))) + ########################### + # train with real + netD.zero_grad() + real_cpu = lite.to_device(data[0]) + batch_size = real_cpu.size(0) + label = torch.full((batch_size,), real_label, dtype=real_cpu.dtype, device=lite.device) + + output = netD(real_cpu) + errD_real = criterion(output, label) + errD_real.backward() + D_x = output.mean().item() + + # train with fake + noise = torch.randn(batch_size, nz, 1, 1, device=lite.device) + fake = netG(noise) + label.fill_(fake_label) + output = netD(fake.detach()) + errD_fake = criterion(output, label) + errD_fake.backward() + D_G_z1 = output.mean().item() + errD = errD_real + errD_fake + optimizerD.step() + + ############################ + # (2) Update G network: maximize log(D(G(z))) + ########################### + netG.zero_grad() + label.fill_(real_label) # fake labels are real for generator cost + output = netD(fake) + errG = criterion(output, label) + errG.backward() + D_G_z2 = output.mean().item() + optimizerG.step() + + print( + "[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)): %.4f / %.4f" + % ( + epoch, + opt.niter, + i, + len(dataloader), + errD.item(), + errG.item(), + D_x, + D_G_z1, + D_G_z2, + ) + ) + if i % 100 == 0: + vutils.save_image(real_cpu, "%s/real_samples.png" % opt.outf, normalize=True) + fake = netG(fixed_noise) + vutils.save_image( + fake.detach(), + "%s/fake_samples_epoch_%03d.png" % (opt.outf, epoch), + normalize=True, + ) + # do checkpointing + torch.save(netG.state_dict(), "%s/netG_epoch_%d.pth" % (opt.outf, epoch)) + torch.save(netD.state_dict(), "%s/netD_epoch_%d.pth" % (opt.outf, epoch)) + + +if __name__ == "__main__": + lite = Lite(accelerator="ddp", num_processes=2) + lite.run() diff --git a/pl_examples/automator_examples/models.py b/pl_examples/lite_examples/models.py similarity index 100% rename from pl_examples/automator_examples/models.py rename to pl_examples/lite_examples/models.py diff --git a/pytorch_lightning/accelerators/accelerator.py b/pytorch_lightning/accelerators/accelerator.py index 4bc3cb158e714..18a3839342e30 100644 --- a/pytorch_lightning/accelerators/accelerator.py +++ b/pytorch_lightning/accelerators/accelerator.py @@ -84,10 +84,10 @@ def setup(self, trainer: "pl.Trainer") -> None: def setup_dataloader(self, dataloader: DataLoader) -> DataLoader: return self.training_type_plugin.setup_dataloader(dataloader) - def setup_model(self, model: nn.Module) -> nn.Module: + def setup_model(self, model: Module) -> Module: return self.training_type_plugin.setup_model_and_optimizers(model, None) - def start_training(self, trainer: 'Trainer') -> None: + def start_training(self, trainer: "Trainer") -> None: self.training_type_plugin.start_training(trainer) def start_evaluating(self, trainer: "pl.Trainer") -> None: @@ -250,7 +250,7 @@ def backward(self, closure_loss: Tensor, *args: Any, **kwargs: Any) -> Tensor: return closure_loss def run_backward(self, tensor: Tensor, *args, **kwargs) -> None: - """ Lightning-independent backward logic """ + """Lightning-independent backward logic""" # TODO: Q: We don't need training_type.pre_/post_backward here right? Because we can't automate # the blocking of "require_backward_grad_sync" for the PyTorch user self.precision_plugin.run_backward(tensor, *args, **kwargs) @@ -269,9 +269,7 @@ def optimizer_step(self, optimizer: Optimizer, opt_idx: int, lambda_closure: Cal if make_optimizer_step: self.run_optimizer_step(optimizer, opt_idx, lambda_closure, **kwargs) - def run_optimizer_step( - self, optimizer: Optimizer, lambda_closure: Callable, **kwargs: Any - ) -> None: + def run_optimizer_step(self, optimizer: Optimizer, lambda_closure: Callable, **kwargs: Any) -> None: """Lightning-independent optimizer step logic""" self.precision_plugin.run_pre_optimizer_step(optimizer) self.training_type_plugin.optimizer_step(optimizer, lambda_closure=lambda_closure, **kwargs) @@ -508,7 +506,7 @@ def on_train_batch_start(self, batch: Any, batch_idx: int, dataloader_idx: int = """Called in the training loop before anything happens for that batch.""" return self.training_type_plugin.on_train_batch_start(batch, batch_idx) - @contextmanager + @contextlib.contextmanager def forward_context(self): with self.precision_plugin.forward_context(), self.training_type_plugin.forward_context(): yield diff --git a/pytorch_lightning/plugins/precision/native_amp.py b/pytorch_lightning/plugins/precision/native_amp.py index baa22900b8398..420f80c88a4b0 100644 --- a/pytorch_lightning/plugins/precision/native_amp.py +++ b/pytorch_lightning/plugins/precision/native_amp.py @@ -99,7 +99,7 @@ def pre_optimizer_step( self.scaler.update() return False - def post_optimizer_step(self, optimizer: 'Optimizer', optimizer_idx: int) -> None: + def post_optimizer_step(self, optimizer: "Optimizer", optimizer_idx: int) -> None: """Updates the GradScaler""" self.run_post_optimizer_step(optimizer) @@ -109,7 +109,7 @@ def run_pre_optimizer_step(self, optimizer: "Optimizer") -> None: def run_post_optimizer_step(self, optimizer: "Optimizer") -> None: self.scaler.step(optimizer) self.scaler.update() - + def autocast_context_manager(self) -> torch.cuda.amp.autocast: if self.use_cpu: return torch.cpu.amp.autocast(dtype=self._dtype) # Only reached in pytorch==1.10 where this is ok. skipcq @@ -121,8 +121,9 @@ def autocast_context_manager(self) -> torch.cuda.amp.autocast: def forward_context(self) -> Generator[None, None, None]: """Enable autocast context""" with torch.cuda.amp.autocast(): - - @contextmanager + yield + + @contextmanager def train_step_context(self) -> Generator[None, None, None]: """Enable autocast context.""" with self.autocast_context_manager(): diff --git a/pytorch_lightning/plugins/training_type/ddp.py b/pytorch_lightning/plugins/training_type/ddp.py index 1fb9ea6ec26dd..6e10ec93495f8 100644 --- a/pytorch_lightning/plugins/training_type/ddp.py +++ b/pytorch_lightning/plugins/training_type/ddp.py @@ -27,7 +27,9 @@ import numpy as np import torch import torch.distributed +from torch.nn import Module from torch.nn.parallel.distributed import DistributedDataParallel +from torch.utils.data import DataLoader, DistributedSampler import pytorch_lightning as pl from pytorch_lightning.core.optimizer import LightningOptimizer @@ -181,7 +183,7 @@ def setup_environment(self) -> None: self.setup_distributed() - def setup_model(self, model: nn.Module) -> nn.Module: + def setup_model(self, model: Module) -> Module: model = DistributedDataParallel( module=model.to(self.root_device), device_ids=self.determine_ddp_device_ids(), @@ -192,7 +194,7 @@ def setup_model(self, model: nn.Module) -> nn.Module: def setup_dataloader(self, dataloader: DataLoader) -> DataLoader: kwargs = self.distributed_sampler_kwargs sampler = DistributedSampler(dataloader.dataset, **kwargs) - dataloader = replace_sampler(dataloader, sampler) + # dataloader = replace_sampler(dataloader, sampler) return dataloader def _call_children_scripts(self): diff --git a/pytorch_lightning/plugins/training_type/tpu_spawn.py b/pytorch_lightning/plugins/training_type/tpu_spawn.py index 45705f65549ad..b0b22438940b5 100644 --- a/pytorch_lightning/plugins/training_type/tpu_spawn.py +++ b/pytorch_lightning/plugins/training_type/tpu_spawn.py @@ -15,11 +15,12 @@ import os import re import time -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union, Callable import torch import torch.multiprocessing as mp from torch.nn import Module +from torch.optim import Optimizer from torch.utils.data import DataLoader import pytorch_lightning as pl @@ -269,7 +270,7 @@ def get_mp_spawn_kwargs(self, trainer: "pl.Trainer") -> dict: } def optimizer_step(self, optimizer: Optimizer, lambda_closure: Callable, **kwargs): - xm.optimizer_step(optimizer, barrier=False, optimizer_args={'closure': lambda_closure, **kwargs}) + xm.optimizer_step(optimizer, barrier=False, optimizer_args={"closure": lambda_closure, **kwargs}) def start_training(self, trainer: "pl.Trainer") -> None: # todo: precision pluging is call in accelerator setup and should be moved diff --git a/pytorch_lightning/plugins/training_type/training_type_plugin.py b/pytorch_lightning/plugins/training_type/training_type_plugin.py index b111fbcbd6d93..d56dc4968a8e0 100644 --- a/pytorch_lightning/plugins/training_type/training_type_plugin.py +++ b/pytorch_lightning/plugins/training_type/training_type_plugin.py @@ -13,11 +13,12 @@ # limitations under the License. import contextlib from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, Generator, Iterable, Mapping, Optional, Union +from typing import Any, Callable, Dict, Generator, Iterable, Mapping, Optional, Union, Sequence import torch from torch import Tensor from torch.nn import Module +from torch.optim import Optimizer from torch.utils.data import DataLoader import pytorch_lightning as pl diff --git a/pytorch_lightning/trainer/connectors/accelerator_connector.py b/pytorch_lightning/trainer/connectors/accelerator_connector.py index 467fa3b898ada..d67df5bf55e44 100644 --- a/pytorch_lightning/trainer/connectors/accelerator_connector.py +++ b/pytorch_lightning/trainer/connectors/accelerator_connector.py @@ -706,8 +706,8 @@ def select_accelerator(self) -> Accelerator: # that we first select training_type_plugin, then precision_plugin accelerator = acc_cls(training_type_plugin=self.training_type_plugin, precision_plugin=self.precision_plugin) # transfer ownership of the plugins to the accelerator - self._training_type_plugin = proxy(self.training_type_plugin) - self._precision_plugin = proxy(self.precision_plugin) + self._training_type_plugin = self.training_type_plugin + self._precision_plugin = self.precision_plugin return accelerator From 9257f7a354a9222248d48d3e8e95362c1af744ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 12 Oct 2021 18:33:13 +0200 Subject: [PATCH 041/331] run --- pl_examples/lite_examples/gan_example.py | 6 ++-- pytorch_lightning/plugins/base_plugin.py | 32 ------------------- .../plugins/precision/precision_plugin.py | 13 +++++--- .../training_type/training_type_plugin.py | 5 +++ 4 files changed, 16 insertions(+), 40 deletions(-) delete mode 100644 pytorch_lightning/plugins/base_plugin.py diff --git a/pl_examples/lite_examples/gan_example.py b/pl_examples/lite_examples/gan_example.py index 197bd2e19f008..1c7136d46d0b3 100644 --- a/pl_examples/lite_examples/gan_example.py +++ b/pl_examples/lite_examples/gan_example.py @@ -73,7 +73,7 @@ def run(self): dataset, batch_size=opt.batchSize, shuffle=True, num_workers=opt.workers ) - dataloader = lite.setup(dataloader) + dataloader = lite.setup_dataloader(dataloader) # assert isinstance(dataloader.sampler, DistributedSampler) netG = Generator() @@ -85,8 +85,6 @@ def run(self): lite.to_device(netG) lite.to_device(netD) - netG, netD = lite.setup(netG, netD) - # assert isinstance(netG, DistributedDataParallel) # assert isinstance(netD, DistributedDataParallel) @@ -100,7 +98,7 @@ def run(self): optimizerD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) - optimizerG, optimizerG = lite.setup(optimizerG, optimizerD) + (netG, netD), (optimizerG, optimizerD) = lite.setup(models=(netG, netD), optimizers=(optimizerG, optimizerD)) for epoch in range(opt.niter): for i, data in enumerate(dataloader, 0): diff --git a/pytorch_lightning/plugins/base_plugin.py b/pytorch_lightning/plugins/base_plugin.py deleted file mode 100644 index d2e7a566d0d75..0000000000000 --- a/pytorch_lightning/plugins/base_plugin.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright The PyTorch Lightning team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import contextlib -from abc import ABC -from typing import Generator - - -class Plugin(ABC): - """Basic Plugin class to derive precision and training type plugins from.""" - - def pre_dispatch(self) -> None: - """Hook to do something before the training/evaluation/prediction starts.""" - - def post_dispatch(self) -> None: - """Hook to do something after the training/evaluation/prediction finishes.""" - - @contextlib.contextmanager - def forward_context(self) -> Generator: - """A contextmanager for managing model forward/training_step/evaluation_step/predict_step""" - yield - diff --git a/pytorch_lightning/plugins/precision/precision_plugin.py b/pytorch_lightning/plugins/precision/precision_plugin.py index 4fd70af76df25..47e7795660bec 100644 --- a/pytorch_lightning/plugins/precision/precision_plugin.py +++ b/pytorch_lightning/plugins/precision/precision_plugin.py @@ -91,7 +91,7 @@ def post_backward(self, model: "pl.LightningModule", closure_loss: Tensor) -> Te return closure_loss def run_backward(self, tensor, *args, **kwargs) -> None: - """ Lightning-independent backward logic. """ + """Lightning-independent backward logic.""" tensor.backward(*args, **kwargs) def pre_optimizer_step( @@ -106,14 +106,14 @@ def pre_optimizer_step( model.trainer.call_hook("on_before_optimizer_step", optimizer, optimizer_idx) return True - def post_optimizer_step(self, optimizer: 'Optimizer', optimizer_idx: int) -> None: + def post_optimizer_step(self, optimizer: "Optimizer", optimizer_idx: int) -> None: """Hook to do something after each optimizer step.""" def run_pre_optimizer_step(self, optimizer: "Optimizer"): - """ Lightning-independent pre optimizer step logic. """ + """Lightning-independent pre optimizer step logic.""" def run_post_optimizer_step(self, optimizer: "Optimizer"): - """ Lightning-independent post optimizer step logic. """ + """Lightning-independent post optimizer step logic.""" def clip_gradients( self, @@ -155,6 +155,11 @@ def dispatch(self, trainer: "pl.Trainer") -> None: def post_dispatch(self) -> None: """Hook to do something after the training/evaluation/prediction finishes.""" + @contextlib.contextmanager + def forward_context(self) -> Generator: + """A contextmanager for managing model forward/training_step/evaluation_step/predict_step""" + yield + @contextlib.contextmanager def train_step_context(self) -> Generator: """A contextmanager for the training step.""" diff --git a/pytorch_lightning/plugins/training_type/training_type_plugin.py b/pytorch_lightning/plugins/training_type/training_type_plugin.py index d56dc4968a8e0..f2f434b0ff089 100644 --- a/pytorch_lightning/plugins/training_type/training_type_plugin.py +++ b/pytorch_lightning/plugins/training_type/training_type_plugin.py @@ -302,6 +302,11 @@ def remove_checkpoint(self, filepath: _PATH) -> None: if self.should_rank_save_checkpoint: return self.checkpoint_io.remove_checkpoint(filepath) + @contextlib.contextmanager + def forward_context(self) -> Generator: + """A contextmanager for managing model forward/training_step/evaluation_step/predict_step""" + yield + @contextlib.contextmanager def model_sharded_context(self) -> Generator: """Provide hook to create modules in a distributed aware context. This is useful for when we'd like to From bd5141414f9b71011aadacb9aaddd82ac9767e95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 12 Oct 2021 18:34:05 +0200 Subject: [PATCH 042/331] formatting --- pl_examples/lite_examples/gan_example.py | 4 +++- pl_examples/lite_examples/models.py | 3 +-- pytorch_lightning/lite/lite.py | 10 ++++------ .../plugins/precision/deepspeed_precision.py | 1 - pytorch_lightning/plugins/training_type/tpu_spawn.py | 2 +- .../plugins/training_type/training_type_plugin.py | 2 +- 6 files changed, 10 insertions(+), 12 deletions(-) diff --git a/pl_examples/lite_examples/gan_example.py b/pl_examples/lite_examples/gan_example.py index 1c7136d46d0b3..92ee2b1982a66 100644 --- a/pl_examples/lite_examples/gan_example.py +++ b/pl_examples/lite_examples/gan_example.py @@ -7,9 +7,11 @@ """ from __future__ import print_function + import argparse import os import random + import torch import torch.nn as nn import torch.nn.parallel @@ -19,7 +21,7 @@ import torchvision.transforms as transforms import torchvision.utils as vutils -from pl_examples.lite_examples.models import weights_init, Generator, Discriminator +from pl_examples.lite_examples.models import Discriminator, Generator, weights_init from pytorch_lightning.lite import LightningLite parser = argparse.ArgumentParser() diff --git a/pl_examples/lite_examples/models.py b/pl_examples/lite_examples/models.py index a54baf38c1052..8f73902ff580c 100644 --- a/pl_examples/lite_examples/models.py +++ b/pl_examples/lite_examples/models.py @@ -1,7 +1,6 @@ import torch from torch import nn as nn - nc = 1 nz = 100 ngf = 64 @@ -75,4 +74,4 @@ def __init__(self): def forward(self, input): print("autocast enabled in discriminator: ", torch.is_autocast_enabled()) output = self.main(input) - return output.view(-1, 1).squeeze(1) \ No newline at end of file + return output.view(-1, 1).squeeze(1) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index d12f18d7bf11b..8e4008b8e9310 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -1,19 +1,17 @@ -from weakref import proxy from collections import Callable from contextlib import contextmanager -from typing import Any, Union, Optional, Sequence, Tuple +from typing import Any, Optional, Sequence, Tuple, Union +from weakref import proxy +import torch.multiprocessing as mp import torch.nn as nn from torch import Tensor from torch.optim import Optimizer from torch.utils.data import DataLoader -import torch.multiprocessing as mp from pytorch_lightning import Trainer from pytorch_lightning.accelerators import Accelerator -from pytorch_lightning.trainer.connectors.accelerator_connector import ( - AcceleratorConnector, -) +from pytorch_lightning.trainer.connectors.accelerator_connector import AcceleratorConnector from pytorch_lightning.utilities import move_data_to_device diff --git a/pytorch_lightning/plugins/precision/deepspeed_precision.py b/pytorch_lightning/plugins/precision/deepspeed_precision.py index 636f8f57c1731..59de0a5ac8d32 100644 --- a/pytorch_lightning/plugins/precision/deepspeed_precision.py +++ b/pytorch_lightning/plugins/precision/deepspeed_precision.py @@ -65,7 +65,6 @@ def backward(self, model: "pl.LightningModule", closure_loss: Tensor, *args: Any deepspeed_engine = model.trainer.model deepspeed_engine.backward(closure_loss, *args, **kwargs) - def run_backward(self, tensor, *args, **kwargs): # TODO: implement pass diff --git a/pytorch_lightning/plugins/training_type/tpu_spawn.py b/pytorch_lightning/plugins/training_type/tpu_spawn.py index b0b22438940b5..73f7467f575cf 100644 --- a/pytorch_lightning/plugins/training_type/tpu_spawn.py +++ b/pytorch_lightning/plugins/training_type/tpu_spawn.py @@ -15,7 +15,7 @@ import os import re import time -from typing import Any, Dict, List, Optional, Union, Callable +from typing import Any, Callable, Dict, List, Optional, Union import torch import torch.multiprocessing as mp diff --git a/pytorch_lightning/plugins/training_type/training_type_plugin.py b/pytorch_lightning/plugins/training_type/training_type_plugin.py index f2f434b0ff089..025b24973789f 100644 --- a/pytorch_lightning/plugins/training_type/training_type_plugin.py +++ b/pytorch_lightning/plugins/training_type/training_type_plugin.py @@ -13,7 +13,7 @@ # limitations under the License. import contextlib from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, Generator, Iterable, Mapping, Optional, Union, Sequence +from typing import Any, Callable, Dict, Generator, Iterable, Mapping, Optional, Sequence, Union import torch from torch import Tensor From af9f6e8e601710c9016508b0ebf1f42701982f4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 13 Oct 2021 01:02:57 +0200 Subject: [PATCH 043/331] update --- pl_examples/lite_examples/gan_example.py | 18 ++-- pytorch_lightning/lite/lite.py | 86 ++++++++----------- pytorch_lightning/lite/wrappers.py | 52 +++++++++++ .../plugins/precision/deepspeed_precision.py | 4 +- 4 files changed, 100 insertions(+), 60 deletions(-) create mode 100644 pytorch_lightning/lite/wrappers.py diff --git a/pl_examples/lite_examples/gan_example.py b/pl_examples/lite_examples/gan_example.py index 92ee2b1982a66..721457d4574ac 100644 --- a/pl_examples/lite_examples/gan_example.py +++ b/pl_examples/lite_examples/gan_example.py @@ -57,7 +57,7 @@ def run(self): # TODO: how do we handle this in Accelerator? # torch.cuda.set_device(opt.local_rank) # TODO: how do we handle this? - os.environ["LOCAL_RANK"] = str(opt.local_rank) + # os.environ["LOCAL_RANK"] = str(opt.local_rank) # os.environ["NODE_RANK"] = str(opt.local_rank) dataset = dset.MNIST( @@ -75,7 +75,7 @@ def run(self): dataset, batch_size=opt.batchSize, shuffle=True, num_workers=opt.workers ) - dataloader = lite.setup_dataloader(dataloader) + dataloader = self.setup_dataloader(dataloader) # assert isinstance(dataloader.sampler, DistributedSampler) netG = Generator() @@ -84,15 +84,15 @@ def run(self): netD = Discriminator() netD.apply(weights_init) - lite.to_device(netG) - lite.to_device(netD) + self.to_device(netG) + self.to_device(netD) # assert isinstance(netG, DistributedDataParallel) # assert isinstance(netD, DistributedDataParallel) criterion = nn.BCELoss() - fixed_noise = torch.randn(opt.batchSize, nz, 1, 1, device=lite.device) + fixed_noise = torch.randn(opt.batchSize, nz, 1, 1, device=self.device) real_label = 1 fake_label = 0 @@ -100,7 +100,7 @@ def run(self): optimizerD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) - (netG, netD), (optimizerG, optimizerD) = lite.setup(models=(netG, netD), optimizers=(optimizerG, optimizerD)) + (netG, netD), (optimizerG, optimizerD) = self.setup(models=(netG, netD), optimizers=(optimizerG, optimizerD)) for epoch in range(opt.niter): for i, data in enumerate(dataloader, 0): @@ -109,9 +109,9 @@ def run(self): ########################### # train with real netD.zero_grad() - real_cpu = lite.to_device(data[0]) + real_cpu = self.to_device(data[0]) batch_size = real_cpu.size(0) - label = torch.full((batch_size,), real_label, dtype=real_cpu.dtype, device=lite.device) + label = torch.full((batch_size,), real_label, dtype=real_cpu.dtype, device=self.device) output = netD(real_cpu) errD_real = criterion(output, label) @@ -119,7 +119,7 @@ def run(self): D_x = output.mean().item() # train with fake - noise = torch.randn(batch_size, nz, 1, 1, device=lite.device) + noise = torch.randn(batch_size, nz, 1, 1, device=self.device) fake = netG(noise) label.fill_(fake_label) output = netD(fake.detach()) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 8e4008b8e9310..6971f5ba93961 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -1,9 +1,22 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import abstractmethod, ABC from collections import Callable from contextlib import contextmanager -from typing import Any, Optional, Sequence, Tuple, Union -from weakref import proxy +from typing import Any, Optional, Sequence, Union, List -import torch.multiprocessing as mp import torch.nn as nn from torch import Tensor from torch.optim import Optimizer @@ -11,57 +24,27 @@ from pytorch_lightning import Trainer from pytorch_lightning.accelerators import Accelerator +from pytorch_lightning.lite.wrappers import _LiteOptimizer, _LiteModel +from pytorch_lightning.plugins import PLUGIN_INPUT from pytorch_lightning.trainer.connectors.accelerator_connector import AcceleratorConnector from pytorch_lightning.utilities import move_data_to_device -class LiteOptimizer(Optimizer): - def __init__(self, optimizer: Optimizer, accelerator: Accelerator): - super().__init__(params=optimizer.param_groups, defaults={}) - self.optimizer = optimizer - self._accelerator = accelerator - - def step(self, closure=None, **kwargs: Any): - print("running automated step") - output = self._accelerator.run_optimizer_step( - self.optimizer, - lambda_closure=closure, - **kwargs, - ) - return output - - -class LiteModel(nn.Module): - def __init__(self, module: nn.Module, accelerator: Accelerator): - super().__init__() - self._module = module - self._accelerator = accelerator - - @property - def module(self): - return self._module - - def forward(self, *args, **kwargs): - with self._accelerator.forward_context(): - output = self.module.forward(*args, **kwargs) - return output - - -class LightningLite: +class LightningLite(ABC): def __init__( self, - accelerator=None, - plugins=None, - gpus=None, - tpu_cores=None, - ipus=None, - num_processes=1, - devices=None, - num_nodes=1, - precision=32, + accelerator: Optional[Union[str, Accelerator]] = None, + plugins: Optional[Union[PLUGIN_INPUT, List[PLUGIN_INPUT]]] = None, + gpus: Optional[Union[List[int], str, int]] = None, + tpu_cores: Optional[Union[List[int], str, int]] = None, + ipus: Optional[int] = None, + num_processes: int = 1, + devices: Optional[Union[List[int], str, int]] = None, + num_nodes: int = 1, + precision: Union[int, str] = 32, amp_backend: str = "native", - amp_level: str = "O2", - replace_sampler_ddp=True, + amp_level: Optional[str] = None, + replace_sampler_ddp: bool = True, ): gpu_ids, tpu_cores = Trainer._parse_devices(gpus=gpus, auto_select_gpus=False, tpu_cores=tpu_cores) backend_connector = AcceleratorConnector( @@ -87,6 +70,7 @@ def __init__( # TODO: Do we need to initialize distributed at the very beginning # any reason to delay?? + # this will also launch processes in ddp/ddp_spawn self.accelerator.setup_environment() @property @@ -102,6 +86,10 @@ def device(self): # the device on the local rank return self.training_type_plugin.root_device + @abstractmethod + def run(self, *args, **kwarg): + pass + def setup( self, models: Union[nn.Module, Sequence[nn.Module]], @@ -119,8 +107,8 @@ def setup( def _setup_models_and_optimizers(self, models: Sequence[nn.Module], optimizers: Sequence[Optimizer]): # Let accelerator/plugin wrap and connect the models and optimizers models, optimizers = self.training_type_plugin.setup_models_and_optimizers(models, optimizers) - models = [LiteModel(module=model, accelerator=self.accelerator) for model in models] - optimizers = [LiteOptimizer(optimizer=optimizer, accelerator=self.accelerator) for optimizer in optimizers] + models = [_LiteModel(module=model, accelerator=self.accelerator) for model in models] + optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self.accelerator) for optimizer in optimizers] return models, optimizers def setup_dataloader(self, *dataloaders: DataLoader): diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py new file mode 100644 index 0000000000000..b03c3f1cc167a --- /dev/null +++ b/pytorch_lightning/lite/wrappers.py @@ -0,0 +1,52 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any + +from torch import nn as nn +from torch.optim import Optimizer + +from pytorch_lightning.accelerators import Accelerator + + +class _LiteOptimizer(Optimizer): + def __init__(self, optimizer: Optimizer, accelerator: Accelerator): + super().__init__(params=optimizer.param_groups, defaults=optimizer.defaults) + self.optimizer = optimizer + self._accelerator = accelerator + + def step(self, closure=None, **kwargs: Any): + print("running automated step") + output = self._accelerator.run_optimizer_step( + self.optimizer, + lambda_closure=closure, + **kwargs, + ) + return output + + +class _LiteModel(nn.Module): + def __init__(self, module: nn.Module, accelerator: Accelerator): + super().__init__() + self._module = module + self._accelerator = accelerator + + @property + def module(self): + return self._module + + def forward(self, *args, **kwargs): + with self._accelerator.forward_context(): + output = self.module.forward(*args, **kwargs) + return output diff --git a/pytorch_lightning/plugins/precision/deepspeed_precision.py b/pytorch_lightning/plugins/precision/deepspeed_precision.py index 59de0a5ac8d32..03be19e143c72 100644 --- a/pytorch_lightning/plugins/precision/deepspeed_precision.py +++ b/pytorch_lightning/plugins/precision/deepspeed_precision.py @@ -66,8 +66,8 @@ def backward(self, model: "pl.LightningModule", closure_loss: Tensor, *args: Any deepspeed_engine.backward(closure_loss, *args, **kwargs) def run_backward(self, tensor, *args, **kwargs): - # TODO: implement - pass + deepspeed_engine = None # TODO: access engine here + deepspeed_engine.backward(tensor, *args, **kwargs) def clip_gradients( self, From 914c6069dc437650f1871b70dbb37f869c0585b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 13 Oct 2021 01:18:18 +0200 Subject: [PATCH 044/331] update --- pytorch_lightning/lite/lite.py | 73 +++++++++++++--------------- pytorch_lightning/trainer/trainer.py | 2 +- 2 files changed, 35 insertions(+), 40 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 6971f5ba93961..ec6b3a5324cc7 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -15,7 +15,8 @@ from abc import abstractmethod, ABC from collections import Callable from contextlib import contextmanager -from typing import Any, Optional, Sequence, Union, List +from pathlib import Path +from typing import Any, Optional, Sequence, Union, List, Dict import torch.nn as nn from torch import Tensor @@ -66,25 +67,33 @@ def __init__( amp_level=amp_level, plugins=plugins, ) - self.accelerator = backend_connector.select_accelerator() - + self._accelerator = backend_connector.select_accelerator() + self._training_type_plugin = self._accelerator.training_type_plugin + self._precision_plugin = self._accelerator.precision_plugin # TODO: Do we need to initialize distributed at the very beginning # any reason to delay?? # this will also launch processes in ddp/ddp_spawn - self.accelerator.setup_environment() + self._accelerator.setup_environment() @property - def training_type_plugin(self): - return self.accelerator.training_type_plugin + def device(self): + return self._accelerator.root_device @property - def precision_plugin(self): - return self.accelerator.precision_plugin + def global_rank(self): + return getattr(self._training_type_plugin, "global_rank", 0) @property - def device(self): - # the device on the local rank - return self.training_type_plugin.root_device + def local_rank(self): + return getattr(self._training_type_plugin, "local_rank", 0) + + @property + def node_rank(self): + return getattr(self._training_type_plugin, "node_rank", 0) + + @property + def world_size(self): + return getattr(self._training_type_plugin, "world_size", 1) @abstractmethod def run(self, *args, **kwarg): @@ -106,30 +115,30 @@ def setup( def _setup_models_and_optimizers(self, models: Sequence[nn.Module], optimizers: Sequence[Optimizer]): # Let accelerator/plugin wrap and connect the models and optimizers - models, optimizers = self.training_type_plugin.setup_models_and_optimizers(models, optimizers) - models = [_LiteModel(module=model, accelerator=self.accelerator) for model in models] - optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self.accelerator) for optimizer in optimizers] + models, optimizers = self._training_type_plugin.setup_models_and_optimizers(models, optimizers) + models = [_LiteModel(module=model, accelerator=self._accelerator) for model in models] + optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self._accelerator) for optimizer in optimizers] return models, optimizers def setup_dataloader(self, *dataloaders: DataLoader): # user can call this method independently instead of the general purpose setup method - dataloaders = [self.training_type_plugin.setup_dataloader(dataloader) for dataloader in dataloaders] + dataloaders = [self._training_type_plugin.setup_dataloader(dataloader) for dataloader in dataloaders] dataloaders = dataloaders[0] if len(dataloaders) == 1 else dataloaders return dataloaders def backward(self, tensor: Tensor, *args, **kwargs): # user will call automator.backward(loss) instead of loss.backward() - self.accelerator.run_backward(tensor, *args, **kwargs) + self._accelerator.run_backward(tensor, *args, **kwargs) @contextmanager def forward_context(self): - with self.accelerator.forward_context(): + with self._accelerator.forward_context(): yield # @contextmanager # def backward_context(self, *args, **kwargs): # yield - # + # @contextmanager def optimizer_step_context(self, model=None, optimizer=None): # necessary for deepspeed + scaling @@ -138,31 +147,17 @@ def optimizer_step_context(self, model=None, optimizer=None): yield optimizer.step = temp - def to_device(self, obj: Union[nn.Module, Tensor]) -> Union[nn.Module, Tensor]: + def to_device(self, obj: Union[nn.Module, Tensor, Any]) -> Union[nn.Module, Tensor, Any]: if isinstance(obj, nn.Module): return obj.to(self.device) return move_data_to_device(obj, device=self.device) - def sync(self, data: Any) -> Any: - # all_gather - pass - - def reduce_data(self, data: Any) -> Any: - return self.training_type_plugin.reduce(data) - def reduce_decision(self, decision: bool) -> bool: - return self.training_type_plugin.reduce_boolean_decision(decision) - - def broadcast_decision(self, decision: bool): - # return self.training_type_plugin.broadcast_boolean_decision(decision) - return False + return self._training_type_plugin.reduce_boolean_decision(decision) - def save_checkpoint(self, filepath): - pass - - def execute_on_rank(self, func: Callable, rank: int): - pass + def save_checkpoint(self, filepath: Union[str, Path], content: Dict[str, Any]): + raise NotImplementedError() - def spawn(self, function: Callable, *args: Any): - # ctx = mp.spawn(function, args, nprocs=..., ...) - pass + def execute_on_rank(self, func: Callable, rank: int, *args: Any, **kwargs: Any) -> None: + if self.global_rank == rank: + func(*args, **kwargs) diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index f3401120130ac..44c830f9aab80 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -1499,7 +1499,7 @@ def local_rank(self) -> int: @property def node_rank(self) -> int: - # some training types define a local rank + # some training types define a node rank return getattr(self.accelerator.training_type_plugin, "node_rank", 0) @property From 8d1d59497ee7ab22da396d98d4e61ff6b0088e1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 13 Oct 2021 01:40:03 +0200 Subject: [PATCH 045/331] examples --- pl_examples/lite_examples/gan/__init__.py | 0 pl_examples/lite_examples/{ => gan}/gan_example.py | 10 +++++----- pl_examples/lite_examples/{ => gan}/models.py | 0 pl_examples/lite_examples/gan/run_examples.py | 14 ++++++++++++++ 4 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 pl_examples/lite_examples/gan/__init__.py rename pl_examples/lite_examples/{ => gan}/gan_example.py (96%) rename pl_examples/lite_examples/{ => gan}/models.py (100%) create mode 100644 pl_examples/lite_examples/gan/run_examples.py diff --git a/pl_examples/lite_examples/gan/__init__.py b/pl_examples/lite_examples/gan/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/pl_examples/lite_examples/gan_example.py b/pl_examples/lite_examples/gan/gan_example.py similarity index 96% rename from pl_examples/lite_examples/gan_example.py rename to pl_examples/lite_examples/gan/gan_example.py index 721457d4574ac..a0f942e7e758a 100644 --- a/pl_examples/lite_examples/gan_example.py +++ b/pl_examples/lite_examples/gan/gan_example.py @@ -21,7 +21,7 @@ import torchvision.transforms as transforms import torchvision.utils as vutils -from pl_examples.lite_examples.models import Discriminator, Generator, weights_init +from pl_examples.lite_examples.gan.models import Discriminator, Generator, weights_init from pytorch_lightning.lite import LightningLite parser = argparse.ArgumentParser() @@ -42,14 +42,14 @@ parser.add_argument("--outf", default="./lightning_logs", help="folder to output images and model checkpoints") parser.add_argument("--local_rank", type=int, default=0) -opt = parser.parse_args() +opt, _ = parser.parse_known_args() os.makedirs(opt.outf, exist_ok=True) ngpu = int(opt.ngpu) nz = 100 -class Lite(LightningLite): +class GANTrainer(LightningLite): def run(self): random.seed(123) torch.manual_seed(123) @@ -168,5 +168,5 @@ def run(self): if __name__ == "__main__": - lite = Lite(accelerator="ddp", num_processes=2) - lite.run() + gan = GANTrainer(accelerator="ddp", num_processes=2) + gan.run() diff --git a/pl_examples/lite_examples/models.py b/pl_examples/lite_examples/gan/models.py similarity index 100% rename from pl_examples/lite_examples/models.py rename to pl_examples/lite_examples/gan/models.py diff --git a/pl_examples/lite_examples/gan/run_examples.py b/pl_examples/lite_examples/gan/run_examples.py new file mode 100644 index 0000000000000..15bebf7933319 --- /dev/null +++ b/pl_examples/lite_examples/gan/run_examples.py @@ -0,0 +1,14 @@ +import argparse + +from pl_examples.lite_examples.gan.gan_example import GANTrainer + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--accelerator", type=str, default=None) + parser.add_argument("--gpus", type=int, default=None) + parser.add_argument("--num_processes", type=int, default=1) + parser.add_argument("--precision", type=int, default=32) + args = parser.parse_args() + + trainer = GANTrainer(**vars(args)) + trainer.run() From 82bc3c1dd366a4fb0756073e4e72eae51058d892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 13 Oct 2021 12:03:59 +0200 Subject: [PATCH 046/331] implement run for spawning processes --- pytorch_lightning/lite/lite.py | 23 ++++++++++++++----- .../plugins/training_type/ddp_spawn.py | 12 +++++++++- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index ec6b3a5324cc7..771719da13772 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -15,6 +15,7 @@ from abc import abstractmethod, ABC from collections import Callable from contextlib import contextmanager +from functools import wraps, partial from pathlib import Path from typing import Any, Optional, Sequence, Union, List, Dict @@ -26,7 +27,7 @@ from pytorch_lightning import Trainer from pytorch_lightning.accelerators import Accelerator from pytorch_lightning.lite.wrappers import _LiteOptimizer, _LiteModel -from pytorch_lightning.plugins import PLUGIN_INPUT +from pytorch_lightning.plugins import PLUGIN_INPUT, DDPSpawnPlugin from pytorch_lightning.trainer.connectors.accelerator_connector import AcceleratorConnector from pytorch_lightning.utilities import move_data_to_device @@ -70,10 +71,9 @@ def __init__( self._accelerator = backend_connector.select_accelerator() self._training_type_plugin = self._accelerator.training_type_plugin self._precision_plugin = self._accelerator.precision_plugin - # TODO: Do we need to initialize distributed at the very beginning - # any reason to delay?? - # this will also launch processes in ddp/ddp_spawn - self._accelerator.setup_environment() + + # wrap the run method so we can inject setup logic or spawn processes for the user + self.run = self._run_wrapper(self.run) @property def device(self): @@ -96,9 +96,20 @@ def world_size(self): return getattr(self._training_type_plugin, "world_size", 1) @abstractmethod - def run(self, *args, **kwarg): + def run(self, *args, **kwarg) -> None: pass + def _run_wrapper(self, run_method: Callable) -> Callable: + return partial(self._run_impl, run_method=run_method) + + def _run_impl(self, run_method, *args, **kwargs) -> None: + self._training_type_plugin.setup_environment() + if isinstance(self._training_type_plugin, DDPSpawnPlugin): + self._training_type_plugin.spawn(run_method, *args, **kwargs) + else: + run_method(*args, **kwargs) + # TODO: any teardown needed here? + def setup( self, models: Union[nn.Module, Sequence[nn.Module]], diff --git a/pytorch_lightning/plugins/training_type/ddp_spawn.py b/pytorch_lightning/plugins/training_type/ddp_spawn.py index eb1acaec4100b..8f002a40aaf10 100644 --- a/pytorch_lightning/plugins/training_type/ddp_spawn.py +++ b/pytorch_lightning/plugins/training_type/ddp_spawn.py @@ -14,8 +14,9 @@ import logging import os import re +from functools import partial from multiprocessing.queues import SimpleQueue -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union, Callable import numpy as np import torch @@ -169,6 +170,15 @@ def start_evaluating(self, trainer: "pl.Trainer") -> None: def start_predicting(self, trainer: "pl.Trainer") -> None: mp.spawn(self.new_process, **self.get_mp_spawn_kwargs(trainer)) + def spawn(self, function: Callable, *args: Any, **kwargs: Any) -> None: + mp.spawn(self._wrapped_function, args=(function, args, kwargs), nprocs=self.num_processes) + + def _wrapped_function(self, process_idx: int, function: Callable, args: Any, kwargs: Any) -> None: + self.set_world_ranks(process_idx) + rank_zero_only.rank = self.global_rank + init_ddp_connection(self.cluster_environment, self.torch_distributed_backend, self.global_rank, self.world_size) + function(*args, **kwargs) + def new_process(self, process_idx: int, trainer: "pl.Trainer", mp_queue: SimpleQueue) -> None: self.mp_queue = mp_queue From 85c09cd8e48d7e629eef2674b498f4a92e0e5b31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 13 Oct 2021 12:09:46 +0200 Subject: [PATCH 047/331] fix spawn not setting master addr --- pytorch_lightning/plugins/training_type/ddp_spawn.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pytorch_lightning/plugins/training_type/ddp_spawn.py b/pytorch_lightning/plugins/training_type/ddp_spawn.py index 8f002a40aaf10..0b2497149e7a4 100644 --- a/pytorch_lightning/plugins/training_type/ddp_spawn.py +++ b/pytorch_lightning/plugins/training_type/ddp_spawn.py @@ -171,6 +171,7 @@ def start_predicting(self, trainer: "pl.Trainer") -> None: mp.spawn(self.new_process, **self.get_mp_spawn_kwargs(trainer)) def spawn(self, function: Callable, *args: Any, **kwargs: Any) -> None: + os.environ["MASTER_PORT"] = str(self.cluster_environment.master_port()) mp.spawn(self._wrapped_function, args=(function, args, kwargs), nprocs=self.num_processes) def _wrapped_function(self, process_idx: int, function: Callable, args: Any, kwargs: Any) -> None: From 1c35e2f0b535fe84f1b4ed32ae5b405cc72cdcda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 13 Oct 2021 12:25:17 +0200 Subject: [PATCH 048/331] make worker setup function, add todos --- .../plugins/training_type/ddp_spawn.py | 24 +++++++------------ 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/pytorch_lightning/plugins/training_type/ddp_spawn.py b/pytorch_lightning/plugins/training_type/ddp_spawn.py index 0b2497149e7a4..bfa1c51867fb7 100644 --- a/pytorch_lightning/plugins/training_type/ddp_spawn.py +++ b/pytorch_lightning/plugins/training_type/ddp_spawn.py @@ -160,14 +160,17 @@ def get_mp_spawn_kwargs(self, trainer: "pl.Trainer") -> dict: return {"args": (trainer, self.mp_queue), "nprocs": self.num_processes} def start_training(self, trainer: "pl.Trainer") -> None: + # TODO: refactor: call self.spawn() here mp.spawn(self.new_process, **self.get_mp_spawn_kwargs(trainer)) # reset optimizers, since main process is never used for training and thus does not have a valid optim state trainer.optimizers = [] def start_evaluating(self, trainer: "pl.Trainer") -> None: + # TODO: refactor: call self.spawn() here mp.spawn(self.new_process, **self.get_mp_spawn_kwargs(trainer)) def start_predicting(self, trainer: "pl.Trainer") -> None: + # TODO: refactor: call self.spawn() here mp.spawn(self.new_process, **self.get_mp_spawn_kwargs(trainer)) def spawn(self, function: Callable, *args: Any, **kwargs: Any) -> None: @@ -175,29 +178,18 @@ def spawn(self, function: Callable, *args: Any, **kwargs: Any) -> None: mp.spawn(self._wrapped_function, args=(function, args, kwargs), nprocs=self.num_processes) def _wrapped_function(self, process_idx: int, function: Callable, args: Any, kwargs: Any) -> None: - self.set_world_ranks(process_idx) - rank_zero_only.rank = self.global_rank - init_ddp_connection(self.cluster_environment, self.torch_distributed_backend, self.global_rank, self.world_size) + self._worker_setup(process_idx) function(*args, **kwargs) - def new_process(self, process_idx: int, trainer: "pl.Trainer", mp_queue: SimpleQueue) -> None: - self.mp_queue = mp_queue - + def _worker_setup(self, process_idx: int): reset_seed() - self.set_world_ranks(process_idx) - - # set warning rank rank_zero_only.rank = self.global_rank - - # set up server using proc 0's ip address - # try to init for 20 times at max in case ports are taken - # where to store ip_table init_ddp_connection(self.cluster_environment, self.torch_distributed_backend, self.global_rank, self.world_size) - # TODO: we moved it to the trainer.fit after calling pre_dispatch - # ... need to double check that it is the correct place - # self.trainer.call_setup_hook(self.model) + def new_process(self, process_idx: int, trainer: "pl.Trainer", mp_queue: SimpleQueue) -> None: + self._worker_setup(process_idx) + self.mp_queue = mp_queue # move the model to the correct device self.model_to_device() From 00550c01746d54f9db7b68ea54d01ad34349dab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 13 Oct 2021 12:28:15 +0200 Subject: [PATCH 049/331] revert setup() changes --- pytorch_lightning/plugins/training_type/training_type_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/plugins/training_type/training_type_plugin.py b/pytorch_lightning/plugins/training_type/training_type_plugin.py index 025b24973789f..4f6d4c24b6668 100644 --- a/pytorch_lightning/plugins/training_type/training_type_plugin.py +++ b/pytorch_lightning/plugins/training_type/training_type_plugin.py @@ -58,7 +58,7 @@ def setup_environment(self) -> None: environment before setup is complete. """ - def setup(self, model: Module) -> None: + def setup(self) -> None: """Called by the accelerator to finish setup.""" def setup_dataloader(self, dataloader: DataLoader) -> DataLoader: From c2a6a7592fa36cc817368e015218e215c64cd801 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 13 Oct 2021 13:05:32 +0200 Subject: [PATCH 050/331] use self.backward --- pl_examples/lite_examples/gan/gan_example.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pl_examples/lite_examples/gan/gan_example.py b/pl_examples/lite_examples/gan/gan_example.py index a0f942e7e758a..973ff476b0d3e 100644 --- a/pl_examples/lite_examples/gan/gan_example.py +++ b/pl_examples/lite_examples/gan/gan_example.py @@ -51,6 +51,7 @@ class GANTrainer(LightningLite): def run(self): + print("selected plugin: ", self._training_type_plugin) random.seed(123) torch.manual_seed(123) @@ -115,7 +116,7 @@ def run(self): output = netD(real_cpu) errD_real = criterion(output, label) - errD_real.backward() + self.backward(errD_real) D_x = output.mean().item() # train with fake @@ -124,7 +125,7 @@ def run(self): label.fill_(fake_label) output = netD(fake.detach()) errD_fake = criterion(output, label) - errD_fake.backward() + self.backward(errD_fake) D_G_z1 = output.mean().item() errD = errD_real + errD_fake optimizerD.step() @@ -136,7 +137,7 @@ def run(self): label.fill_(real_label) # fake labels are real for generator cost output = netD(fake) errG = criterion(output, label) - errG.backward() + self.backward(errG) D_G_z2 = output.mean().item() optimizerG.step() From 085e097161dfa88edde87694c5dac760fe287cc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 13 Oct 2021 13:08:03 +0200 Subject: [PATCH 051/331] add assertions --- pl_examples/lite_examples/gan/gan_example.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pl_examples/lite_examples/gan/gan_example.py b/pl_examples/lite_examples/gan/gan_example.py index 973ff476b0d3e..3bfe396288a1a 100644 --- a/pl_examples/lite_examples/gan/gan_example.py +++ b/pl_examples/lite_examples/gan/gan_example.py @@ -23,6 +23,7 @@ from pl_examples.lite_examples.gan.models import Discriminator, Generator, weights_init from pytorch_lightning.lite import LightningLite +from pytorch_lightning.lite.wrappers import _LiteOptimizer, _LiteModel parser = argparse.ArgumentParser() parser.add_argument("--workers", type=int, help="number of data loading workers", default=0) @@ -103,6 +104,9 @@ def run(self): (netG, netD), (optimizerG, optimizerD) = self.setup(models=(netG, netD), optimizers=(optimizerG, optimizerD)) + assert isinstance(optimizerG, _LiteOptimizer) + assert isinstance(netG, _LiteModel) + for epoch in range(opt.niter): for i, data in enumerate(dataloader, 0): ############################ From 521e355aef9ccd89641364372614acf300a6a2cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 13 Oct 2021 18:16:13 +0200 Subject: [PATCH 052/331] debug --- pl_examples/lite_examples/gan/gan_example.py | 3 + pl_examples/lite_examples/simple/__init__.py | 0 .../lite_examples/simple/mnist_example.py | 150 ++++++++++++++++++ pytorch_lightning/accelerators/accelerator.py | 18 ++- pytorch_lightning/lite/lite.py | 8 +- pytorch_lightning/lite/wrappers.py | 12 +- .../plugins/precision/native_amp.py | 9 +- .../plugins/precision/precision_plugin.py | 5 +- 8 files changed, 184 insertions(+), 21 deletions(-) create mode 100644 pl_examples/lite_examples/simple/__init__.py create mode 100644 pl_examples/lite_examples/simple/mnist_example.py diff --git a/pl_examples/lite_examples/gan/gan_example.py b/pl_examples/lite_examples/gan/gan_example.py index 3bfe396288a1a..87f18b9ee12ca 100644 --- a/pl_examples/lite_examples/gan/gan_example.py +++ b/pl_examples/lite_examples/gan/gan_example.py @@ -119,6 +119,7 @@ def run(self): label = torch.full((batch_size,), real_label, dtype=real_cpu.dtype, device=self.device) output = netD(real_cpu) + print(output.dtype, label.dtype, output.max(), output.min()) errD_real = criterion(output, label) self.backward(errD_real) D_x = output.mean().item() @@ -128,6 +129,7 @@ def run(self): fake = netG(noise) label.fill_(fake_label) output = netD(fake.detach()) + print(output.dtype, label.dtype, output.max(), output.min()) errD_fake = criterion(output, label) self.backward(errD_fake) D_G_z1 = output.mean().item() @@ -140,6 +142,7 @@ def run(self): netG.zero_grad() label.fill_(real_label) # fake labels are real for generator cost output = netD(fake) + print(output.dtype, label.dtype, output.max(), output.min()) errG = criterion(output, label) self.backward(errG) D_G_z2 = output.mean().item() diff --git a/pl_examples/lite_examples/simple/__init__.py b/pl_examples/lite_examples/simple/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/pl_examples/lite_examples/simple/mnist_example.py b/pl_examples/lite_examples/simple/mnist_example.py new file mode 100644 index 0000000000000..396b3720538a5 --- /dev/null +++ b/pl_examples/lite_examples/simple/mnist_example.py @@ -0,0 +1,150 @@ +import argparse +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torchvision import datasets, transforms +from torch.optim.lr_scheduler import StepLR + +from pytorch_lightning.lite import LightningLite + + +class Net(nn.Module): + def __init__(self): + super(Net, self).__init__() + self.conv1 = nn.Conv2d(1, 32, 3, 1) + self.conv2 = nn.Conv2d(32, 64, 3, 1) + self.dropout1 = nn.Dropout(0.25) + self.dropout2 = nn.Dropout(0.5) + self.fc1 = nn.Linear(9216, 128) + self.fc2 = nn.Linear(128, 10) + + def forward(self, x): + x = self.conv1(x) + x = F.relu(x) + x = self.conv2(x) + x = F.relu(x) + x = F.max_pool2d(x, 2) + x = self.dropout1(x) + x = torch.flatten(x, 1) + x = self.fc1(x) + x = F.relu(x) + x = self.dropout2(x) + x = self.fc2(x) + output = F.log_softmax(x, dim=1) + return output + + +class MNIST(LightningLite): + def run(self, args): + use_cuda = self.device.type == "cuda" + + torch.manual_seed(args.seed) + + train_kwargs = {"batch_size": args.batch_size} + test_kwargs = {"batch_size": args.test_batch_size} + if use_cuda: + cuda_kwargs = {"num_workers": 1, "pin_memory": True, "shuffle": True} + train_kwargs.update(cuda_kwargs) + test_kwargs.update(cuda_kwargs) + + transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]) + dataset1 = datasets.MNIST("../data", train=True, download=True, transform=transform) + dataset2 = datasets.MNIST("../data", train=False, transform=transform) + train_loader = torch.utils.data.DataLoader(dataset1, **train_kwargs) + test_loader = torch.utils.data.DataLoader(dataset2, **test_kwargs) + + model = Net().to(self.device) + optimizer = optim.Adadelta(model.parameters(), lr=args.lr) + + train_loader, test_loader = self.setup_dataloader(train_loader, test_loader) + model, optimizer = self.setup(model, optimizer) + + scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) + for epoch in range(1, args.epochs + 1): + self.train(args, model, train_loader, optimizer, epoch) + self.test(model, test_loader) + scheduler.step() + + if args.save_model: + torch.save(model.state_dict(), "mnist_cnn.pt") + + def train(self, args, model, train_loader, optimizer, epoch): + model.train() + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(self.device), target.to(self.device) + optimizer.zero_grad() + output = model(data) + loss = F.nll_loss(output, target) + self.backward(loss) + optimizer.step() + if batch_idx % args.log_interval == 0: + print( + "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format( + epoch, + batch_idx * len(data), + len(train_loader.dataset), + 100.0 * batch_idx / len(train_loader), + loss.item(), + ) + ) + if args.dry_run: + break + + def test(self, model, test_loader): + model.eval() + test_loss = 0 + correct = 0 + with torch.no_grad(): + for data, target in test_loader: + data, target = data.to(self.device), target.to(self.device) + output = model(data) + test_loss += F.nll_loss(output, target, reduction="sum").item() # sum up batch loss + pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability + correct += pred.eq(target.view_as(pred)).sum().item() + + test_loss /= len(test_loader.dataset) + + print( + "\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n".format( + test_loss, correct, len(test_loader.dataset), 100.0 * correct / len(test_loader.dataset) + ) + ) + + +def main(): + # Training settings + parser = argparse.ArgumentParser(description="PyTorch MNIST Example") + parser.add_argument( + "--batch-size", type=int, default=64, metavar="N", help="input batch size for training (default: 64)" + ) + parser.add_argument( + "--test-batch-size", type=int, default=1000, metavar="N", help="input batch size for testing (default: 1000)" + ) + parser.add_argument("--epochs", type=int, default=14, metavar="N", help="number of epochs to train (default: 14)") + parser.add_argument("--lr", type=float, default=1.0, metavar="LR", help="learning rate (default: 1.0)") + parser.add_argument("--gamma", type=float, default=0.7, metavar="M", help="Learning rate step gamma (default: 0.7)") + parser.add_argument("--dry-run", action="store_true", default=False, help="quickly check a single pass") + parser.add_argument("--seed", type=int, default=1, metavar="S", help="random seed (default: 1)") + parser.add_argument( + "--log-interval", + type=int, + default=10, + metavar="N", + help="how many batches to wait before logging training status", + ) + parser.add_argument("--save-model", action="store_true", default=False, help="For Saving the current Model") + parser.add_argument("--accelerator", type=str, default=None) + parser.add_argument("--gpus", type=int, default=None) + parser.add_argument("--num_processes", type=int, default=1) + parser.add_argument("--precision", type=int, default=32) + args = parser.parse_args() + + mnist = MNIST( + gpus=args.gpus, accelerator=args.accelerator, num_processes=args.num_processes, precision=args.precision + ) + mnist.run(args) + + +if __name__ == "__main__": + main() diff --git a/pytorch_lightning/accelerators/accelerator.py b/pytorch_lightning/accelerators/accelerator.py index 18a3839342e30..5ec88c6c6d27a 100644 --- a/pytorch_lightning/accelerators/accelerator.py +++ b/pytorch_lightning/accelerators/accelerator.py @@ -255,7 +255,14 @@ def run_backward(self, tensor: Tensor, *args, **kwargs) -> None: # the blocking of "require_backward_grad_sync" for the PyTorch user self.precision_plugin.run_backward(tensor, *args, **kwargs) - def optimizer_step(self, optimizer: Optimizer, opt_idx: int, lambda_closure: Callable, **kwargs: Any) -> None: + def optimizer_step( + self, + optimizer: Optimizer, + opt_idx: int = 0, + lambda_closure: Optional[Callable] = None, + model: Optional[Union[Module, "pl.LightningModule"]] = None, + **kwargs: Any + ) -> None: """performs the actual optimizer step. Args: @@ -263,18 +270,15 @@ def optimizer_step(self, optimizer: Optimizer, opt_idx: int, lambda_closure: Cal opt_idx: index of the current optimizer lambda_closure: closure calculating the loss value """ + model = model if model is not None else self.lightning_module make_optimizer_step = self.precision_plugin.pre_optimizer_step( - self.lightning_module, optimizer, opt_idx, lambda_closure, **kwargs + model, optimizer, opt_idx, lambda_closure, **kwargs ) if make_optimizer_step: - self.run_optimizer_step(optimizer, opt_idx, lambda_closure, **kwargs) + self.run_optimizer_step(optimizer, lambda_closure, **kwargs) def run_optimizer_step(self, optimizer: Optimizer, lambda_closure: Callable, **kwargs: Any) -> None: - """Lightning-independent optimizer step logic""" - self.precision_plugin.run_pre_optimizer_step(optimizer) self.training_type_plugin.optimizer_step(optimizer, lambda_closure=lambda_closure, **kwargs) - self.precision_plugin.run_post_optimizer_step(optimizer) - # TODO: do we need to call training_type_plugin.post_optimizer_step? def optimizer_zero_grad(self, current_epoch: int, batch_idx: int, optimizer: Optimizer, opt_idx: int) -> None: """Zeros all model parameter's gradients.""" diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 771719da13772..df0e99f3dd3ed 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -96,11 +96,11 @@ def world_size(self): return getattr(self._training_type_plugin, "world_size", 1) @abstractmethod - def run(self, *args, **kwarg) -> None: + def run(self, *args, **kwargs) -> None: pass def _run_wrapper(self, run_method: Callable) -> Callable: - return partial(self._run_impl, run_method=run_method) + return partial(self._run_impl, run_method) def _run_impl(self, run_method, *args, **kwargs) -> None: self._training_type_plugin.setup_environment() @@ -116,8 +116,8 @@ def setup( optimizers: Union[Optimizer, Sequence[Optimizer]], ): # wrap all objects passed in and return them in the same order - models = [models] if len(models) == 1 else models - optimizers = [optimizers] if len(optimizers) == 1 else optimizers + models = [models] if isinstance(models, nn.Module) or len(models) == 1 else models + optimizers = [optimizers] if isinstance(optimizers, Optimizer) or len(optimizers) == 1 else optimizers models, optimizers = self._setup_models_and_optimizers(models, optimizers) models = models[0] if len(models) == 1 else models diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index b03c3f1cc167a..45dbbf22e5791 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -14,10 +14,12 @@ from typing import Any -from torch import nn as nn +import torch +from torch import nn as nn, Tensor from torch.optim import Optimizer from pytorch_lightning.accelerators import Accelerator +from pytorch_lightning.utilities.apply_func import apply_to_collection class _LiteOptimizer(Optimizer): @@ -26,12 +28,12 @@ def __init__(self, optimizer: Optimizer, accelerator: Accelerator): self.optimizer = optimizer self._accelerator = accelerator - def step(self, closure=None, **kwargs: Any): + def step(self, closure=None): print("running automated step") - output = self._accelerator.run_optimizer_step( + output = self._accelerator.optimizer_step( self.optimizer, lambda_closure=closure, - **kwargs, + model=None, ) return output @@ -49,4 +51,6 @@ def module(self): def forward(self, *args, **kwargs): with self._accelerator.forward_context(): output = self.module.forward(*args, **kwargs) + + output = apply_to_collection(output, function=lambda t: t.to(torch.get_default_dtype()), dtype=Tensor) return output diff --git a/pytorch_lightning/plugins/precision/native_amp.py b/pytorch_lightning/plugins/precision/native_amp.py index 420f80c88a4b0..b5dbf3819f3e0 100644 --- a/pytorch_lightning/plugins/precision/native_amp.py +++ b/pytorch_lightning/plugins/precision/native_amp.py @@ -15,6 +15,7 @@ from typing import Any, Callable, Dict, Generator, Union import torch +from torch.nn import Module from torch.optim import LBFGS, Optimizer import pytorch_lightning as pl @@ -75,7 +76,7 @@ def run_backward(self, tensor, *args, **kwargs): def pre_optimizer_step( self, - model: "pl.LightningModule", + model: Union["pl.LightningModule", Module], optimizer: Optimizer, optimizer_idx: int, lambda_closure: Callable, @@ -88,12 +89,12 @@ def pre_optimizer_step( raise MisconfigurationException( f"Native AMP and the LBFGS optimizer are not compatible (optimizer {optimizer_idx})." ) - result = lambda_closure() # native amp does not support closures + result = lambda_closure() if lambda_closure is not None else None # native amp does not support closures self.scaler.unscale_(optimizer) super().pre_optimizer_step(model, optimizer, optimizer_idx, lambda_closure, **kwargs) skipped_backward = result is None # in manual optimization, the closure does not return a value - if not model.automatic_optimization or not skipped_backward: + if not isinstance(model, pl.LightningModule) or not model.automatic_optimization or not skipped_backward: # note: the scaler will skip the `optimizer.step` if nonfinite gradients are found self.scaler.step(optimizer) self.scaler.update() @@ -120,7 +121,7 @@ def autocast_context_manager(self) -> torch.cuda.amp.autocast: @contextmanager def forward_context(self) -> Generator[None, None, None]: """Enable autocast context""" - with torch.cuda.amp.autocast(): + with self.autocast_context_manager(): yield @contextmanager diff --git a/pytorch_lightning/plugins/precision/precision_plugin.py b/pytorch_lightning/plugins/precision/precision_plugin.py index 47e7795660bec..f8b56d1f746ee 100644 --- a/pytorch_lightning/plugins/precision/precision_plugin.py +++ b/pytorch_lightning/plugins/precision/precision_plugin.py @@ -96,14 +96,15 @@ def run_backward(self, tensor, *args, **kwargs) -> None: def pre_optimizer_step( self, - model: "pl.LightningModule", + model: Union["pl.LightningModule", Module], optimizer: Optimizer, optimizer_idx: int, lambda_closure: Callable, **kwargs: Any, ) -> bool: """Hook to do something before each optimizer step.""" - model.trainer.call_hook("on_before_optimizer_step", optimizer, optimizer_idx) + if isinstance(model, pl.LightningModule): + model.trainer.call_hook("on_before_optimizer_step", optimizer, optimizer_idx) return True def post_optimizer_step(self, optimizer: "Optimizer", optimizer_idx: int) -> None: From 791feea5dd82c1968101e7d5dde8becfdfdd0396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 13 Oct 2021 18:17:35 +0200 Subject: [PATCH 053/331] undo print statements --- pl_examples/lite_examples/gan/gan_example.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pl_examples/lite_examples/gan/gan_example.py b/pl_examples/lite_examples/gan/gan_example.py index 87f18b9ee12ca..3bfe396288a1a 100644 --- a/pl_examples/lite_examples/gan/gan_example.py +++ b/pl_examples/lite_examples/gan/gan_example.py @@ -119,7 +119,6 @@ def run(self): label = torch.full((batch_size,), real_label, dtype=real_cpu.dtype, device=self.device) output = netD(real_cpu) - print(output.dtype, label.dtype, output.max(), output.min()) errD_real = criterion(output, label) self.backward(errD_real) D_x = output.mean().item() @@ -129,7 +128,6 @@ def run(self): fake = netG(noise) label.fill_(fake_label) output = netD(fake.detach()) - print(output.dtype, label.dtype, output.max(), output.min()) errD_fake = criterion(output, label) self.backward(errD_fake) D_G_z1 = output.mean().item() @@ -142,7 +140,6 @@ def run(self): netG.zero_grad() label.fill_(real_label) # fake labels are real for generator cost output = netD(fake) - print(output.dtype, label.dtype, output.max(), output.min()) errG = criterion(output, label) self.backward(errG) D_G_z2 = output.mean().item() From d170976f58ae55f904cdf4cba0fcc3a07d6b8f07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 13 Oct 2021 18:42:13 +0200 Subject: [PATCH 054/331] fix import --- tests/trainer/test_dataloaders.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/trainer/test_dataloaders.py b/tests/trainer/test_dataloaders.py index 012dba041a6f1..9a3c79bdd3cf6 100644 --- a/tests/trainer/test_dataloaders.py +++ b/tests/trainer/test_dataloaders.py @@ -27,7 +27,6 @@ from pytorch_lightning import Callback, seed_everything, Trainer from pytorch_lightning.callbacks import ModelCheckpoint from pytorch_lightning.utilities.data import has_iterable_dataset, has_len -from pytorch_lightning.utilities.distributed import replace_sampler from pytorch_lightning.utilities.exceptions import MisconfigurationException from tests.base import EvalModelTemplate from tests.helpers.boring_model import BoringModel, RandomDataset, RandomIterableDataset, RandomIterableDatasetWithLen From 181ba3d138bcea8f164b23127c4f2ef18ed52d50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 13 Oct 2021 19:03:17 +0200 Subject: [PATCH 055/331] undo merge conflict --- pytorch_lightning/plugins/training_type/dp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/plugins/training_type/dp.py b/pytorch_lightning/plugins/training_type/dp.py index 73860de554077..a5a346f82698c 100644 --- a/pytorch_lightning/plugins/training_type/dp.py +++ b/pytorch_lightning/plugins/training_type/dp.py @@ -54,8 +54,8 @@ def world_size(self) -> int: def setup(self) -> None: # model needs to be moved to the device before it is wrapped - model.to(self.root_device) - self._model = self.setup_model(LightningParallelModule(model)) + self.model_to_device() + self._model = self.setup_model(LightningParallelModule(self._model)) def setup_model(self, model: Module) -> Module: return DataParallel(module=model, device_ids=self.parallel_devices) From 58a465af512124d153003359469480bf00c106b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 13 Oct 2021 19:13:38 +0200 Subject: [PATCH 056/331] add strategy argument --- pytorch_lightning/lite/lite.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index df0e99f3dd3ed..5478ed23107ec 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -27,7 +27,7 @@ from pytorch_lightning import Trainer from pytorch_lightning.accelerators import Accelerator from pytorch_lightning.lite.wrappers import _LiteOptimizer, _LiteModel -from pytorch_lightning.plugins import PLUGIN_INPUT, DDPSpawnPlugin +from pytorch_lightning.plugins import PLUGIN_INPUT, DDPSpawnPlugin, TrainingTypePlugin from pytorch_lightning.trainer.connectors.accelerator_connector import AcceleratorConnector from pytorch_lightning.utilities import move_data_to_device @@ -36,6 +36,7 @@ class LightningLite(ABC): def __init__( self, accelerator: Optional[Union[str, Accelerator]] = None, + strategy: Optional[Union[str, TrainingTypePlugin]] = None, plugins: Optional[Union[PLUGIN_INPUT, List[PLUGIN_INPUT]]] = None, gpus: Optional[Union[List[int], str, int]] = None, tpu_cores: Optional[Union[List[int], str, int]] = None, @@ -56,6 +57,7 @@ def __init__( ipus=ipus, distributed_backend=None, # TODO: remove accelerator=accelerator, + strategy=strategy, gpus=gpus, gpu_ids=gpu_ids, num_nodes=num_nodes, From 14cb1790def2969713eacda99386d0c051e8a733 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 13 Oct 2021 21:24:54 +0200 Subject: [PATCH 057/331] launch -> run Co-authored-by: thomas chaton --- pl_examples/lite_examples/gan/gan_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/lite_examples/gan/gan_example.py b/pl_examples/lite_examples/gan/gan_example.py index 3bfe396288a1a..e19a0f271a24d 100644 --- a/pl_examples/lite_examples/gan/gan_example.py +++ b/pl_examples/lite_examples/gan/gan_example.py @@ -3,7 +3,7 @@ Launch it with this command: -python -m torch.distributed.launch --nproc_per_node=2 gan_example.py +python -m torch.distributed.run --nproc_per_node=2 gan_example.py """ from __future__ import print_function From 92fb85b6d24effb815e7c51568b9ddd7968ffbf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 13 Oct 2021 22:20:33 +0200 Subject: [PATCH 058/331] typing --- pl_examples/lite_examples/gan/gan_example.py | 4 +- pyproject.toml | 1 + pytorch_lightning/lite/lite.py | 61 ++++++++++---------- pytorch_lightning/lite/wrappers.py | 26 ++++----- 4 files changed, 46 insertions(+), 46 deletions(-) diff --git a/pl_examples/lite_examples/gan/gan_example.py b/pl_examples/lite_examples/gan/gan_example.py index e19a0f271a24d..c4a5e34d97d22 100644 --- a/pl_examples/lite_examples/gan/gan_example.py +++ b/pl_examples/lite_examples/gan/gan_example.py @@ -23,7 +23,7 @@ from pl_examples.lite_examples.gan.models import Discriminator, Generator, weights_init from pytorch_lightning.lite import LightningLite -from pytorch_lightning.lite.wrappers import _LiteOptimizer, _LiteModel +from pytorch_lightning.lite.wrappers import _LiteOptimizer, _LiteModule parser = argparse.ArgumentParser() parser.add_argument("--workers", type=int, help="number of data loading workers", default=0) @@ -105,7 +105,7 @@ def run(self): (netG, netD), (optimizerG, optimizerD) = self.setup(models=(netG, netD), optimizers=(optimizerG, optimizerD)) assert isinstance(optimizerG, _LiteOptimizer) - assert isinstance(netG, _LiteModel) + assert isinstance(netG, _LiteModule) for epoch in range(opt.niter): for i, data in enumerate(dataloader, 0): diff --git a/pyproject.toml b/pyproject.toml index be5b5fe4c571a..38a18af2bf218 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ module = [ "pytorch_lightning.callbacks.model_summary", "pytorch_lightning.callbacks.pruning", "pytorch_lightning.callbacks.rich_model_summary", + "pytorch_lightning.lite", "pytorch_lightning.loops.optimization.*", "pytorch_lightning.loops.evaluation_loop", "pytorch_lightning.trainer.connectors.checkpoint_connector", diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 5478ed23107ec..6aa18390ba5e1 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -15,10 +15,11 @@ from abc import abstractmethod, ABC from collections import Callable from contextlib import contextmanager -from functools import wraps, partial +from functools import partial from pathlib import Path -from typing import Any, Optional, Sequence, Union, List, Dict +from typing import Any, Optional, Sequence, Union, List, Dict, Tuple, Generator +import torch import torch.nn as nn from torch import Tensor from torch.optim import Optimizer @@ -26,7 +27,7 @@ from pytorch_lightning import Trainer from pytorch_lightning.accelerators import Accelerator -from pytorch_lightning.lite.wrappers import _LiteOptimizer, _LiteModel +from pytorch_lightning.lite.wrappers import _LiteOptimizer, _LiteModule from pytorch_lightning.plugins import PLUGIN_INPUT, DDPSpawnPlugin, TrainingTypePlugin from pytorch_lightning.trainer.connectors.accelerator_connector import AcceleratorConnector from pytorch_lightning.utilities import move_data_to_device @@ -48,7 +49,7 @@ def __init__( amp_backend: str = "native", amp_level: Optional[str] = None, replace_sampler_ddp: bool = True, - ): + ) -> None: gpu_ids, tpu_cores = Trainer._parse_devices(gpus=gpus, auto_select_gpus=False, tpu_cores=tpu_cores) backend_connector = AcceleratorConnector( num_processes=num_processes, @@ -75,36 +76,36 @@ def __init__( self._precision_plugin = self._accelerator.precision_plugin # wrap the run method so we can inject setup logic or spawn processes for the user - self.run = self._run_wrapper(self.run) + setattr(self, "run", self._run_wrapper(self.run)) @property - def device(self): + def device(self) -> torch.device: return self._accelerator.root_device @property - def global_rank(self): + def global_rank(self) -> int: return getattr(self._training_type_plugin, "global_rank", 0) @property - def local_rank(self): + def local_rank(self) -> int: return getattr(self._training_type_plugin, "local_rank", 0) @property - def node_rank(self): + def node_rank(self) -> int: return getattr(self._training_type_plugin, "node_rank", 0) @property - def world_size(self): + def world_size(self) -> int: return getattr(self._training_type_plugin, "world_size", 1) @abstractmethod - def run(self, *args, **kwargs) -> None: + def run(self, *args: Any, **kwargs: Any) -> None: pass def _run_wrapper(self, run_method: Callable) -> Callable: return partial(self._run_impl, run_method) - def _run_impl(self, run_method, *args, **kwargs) -> None: + def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> None: self._training_type_plugin.setup_environment() if isinstance(self._training_type_plugin, DDPSpawnPlugin): self._training_type_plugin.spawn(run_method, *args, **kwargs) @@ -116,49 +117,49 @@ def setup( self, models: Union[nn.Module, Sequence[nn.Module]], optimizers: Union[Optimizer, Sequence[Optimizer]], - ): + ) -> Tuple[Union[nn.Module, Sequence[nn.Module]], Union[Optimizer, Sequence[Optimizer]]]: # wrap all objects passed in and return them in the same order - models = [models] if isinstance(models, nn.Module) or len(models) == 1 else models - optimizers = [optimizers] if isinstance(optimizers, Optimizer) or len(optimizers) == 1 else optimizers + models = [models] if isinstance(models, nn.Module) else models + optimizers = [optimizers] if isinstance(optimizers, Optimizer) else optimizers models, optimizers = self._setup_models_and_optimizers(models, optimizers) models = models[0] if len(models) == 1 else models optimizers = optimizers[0] if len(optimizers) == 1 else optimizers return models, optimizers - def _setup_models_and_optimizers(self, models: Sequence[nn.Module], optimizers: Sequence[Optimizer]): + def _setup_models_and_optimizers( + self, + models: Sequence[nn.Module], + optimizers: Sequence[Optimizer], + ) -> Tuple[Sequence[_LiteModule], Sequence[_LiteOptimizer]]: # Let accelerator/plugin wrap and connect the models and optimizers models, optimizers = self._training_type_plugin.setup_models_and_optimizers(models, optimizers) - models = [_LiteModel(module=model, accelerator=self._accelerator) for model in models] + models = [_LiteModule(module=model, accelerator=self._accelerator) for model in models] optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self._accelerator) for optimizer in optimizers] return models, optimizers - def setup_dataloader(self, *dataloaders: DataLoader): + def setup_dataloader(self, *dataloaders: DataLoader) -> Union[DataLoader, Sequence[DataLoader]]: # user can call this method independently instead of the general purpose setup method dataloaders = [self._training_type_plugin.setup_dataloader(dataloader) for dataloader in dataloaders] dataloaders = dataloaders[0] if len(dataloaders) == 1 else dataloaders return dataloaders - def backward(self, tensor: Tensor, *args, **kwargs): + def backward(self, tensor: Tensor, *args: Any, **kwargs: Any) -> None: # user will call automator.backward(loss) instead of loss.backward() self._accelerator.run_backward(tensor, *args, **kwargs) @contextmanager - def forward_context(self): + def forward_context(self) -> Generator[None, None, None]: with self._accelerator.forward_context(): yield # @contextmanager - # def backward_context(self, *args, **kwargs): + # def optimizer_step_context(self, model=None, optimizer=None): + # # necessary for deepspeed + scaling + # temp = optimizer.step + # optimizer.step = model.step # yield - - # @contextmanager - def optimizer_step_context(self, model=None, optimizer=None): - # necessary for deepspeed + scaling - temp = optimizer.step - optimizer.step = model.step - yield - optimizer.step = temp + # optimizer.step = temp def to_device(self, obj: Union[nn.Module, Tensor, Any]) -> Union[nn.Module, Tensor, Any]: if isinstance(obj, nn.Module): @@ -168,7 +169,7 @@ def to_device(self, obj: Union[nn.Module, Tensor, Any]) -> Union[nn.Module, Tens def reduce_decision(self, decision: bool) -> bool: return self._training_type_plugin.reduce_boolean_decision(decision) - def save_checkpoint(self, filepath: Union[str, Path], content: Dict[str, Any]): + def save_checkpoint(self, filepath: Union[str, Path], content: Dict[str, Any]) -> None: raise NotImplementedError() def execute_on_rank(self, func: Callable, rank: int, *args: Any, **kwargs: Any) -> None: diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 45dbbf22e5791..19bb8d34c44c1 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -11,8 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from typing import Any +from typing import Any, Callable, Optional import torch from torch import nn as nn, Tensor @@ -23,32 +22,31 @@ class _LiteOptimizer(Optimizer): - def __init__(self, optimizer: Optimizer, accelerator: Accelerator): - super().__init__(params=optimizer.param_groups, defaults=optimizer.defaults) - self.optimizer = optimizer + def __init__(self, optimizer: Optimizer, accelerator: Accelerator) -> None: + super().__init__(params=optimizer.param_groups, defaults=optimizer.defaults) # type: ignore[call-arg] + self.__dict__ = optimizer.__dict__ + self._optimizer = optimizer self._accelerator = accelerator - def step(self, closure=None): - print("running automated step") - output = self._accelerator.optimizer_step( - self.optimizer, + def step(self, closure: Optional[Callable] = None) -> None: + self._accelerator.optimizer_step( + self._optimizer, lambda_closure=closure, model=None, ) - return output -class _LiteModel(nn.Module): - def __init__(self, module: nn.Module, accelerator: Accelerator): +class _LiteModule(nn.Module): + def __init__(self, module: nn.Module, accelerator: Accelerator) -> None: super().__init__() self._module = module self._accelerator = accelerator @property - def module(self): + def module(self) -> nn.Module: return self._module - def forward(self, *args, **kwargs): + def forward(self, *args: Any, **kwargs: Any) -> Any: with self._accelerator.forward_context(): output = self.module.forward(*args, **kwargs) From c19380ae85448ba5e65e2b2b2d23e668082afcb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 13 Oct 2021 22:20:42 +0200 Subject: [PATCH 059/331] comments --- pytorch_lightning/accelerators/accelerator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pytorch_lightning/accelerators/accelerator.py b/pytorch_lightning/accelerators/accelerator.py index 5ec88c6c6d27a..db033eae3f5bd 100644 --- a/pytorch_lightning/accelerators/accelerator.py +++ b/pytorch_lightning/accelerators/accelerator.py @@ -251,8 +251,6 @@ def backward(self, closure_loss: Tensor, *args: Any, **kwargs: Any) -> Tensor: def run_backward(self, tensor: Tensor, *args, **kwargs) -> None: """Lightning-independent backward logic""" - # TODO: Q: We don't need training_type.pre_/post_backward here right? Because we can't automate - # the blocking of "require_backward_grad_sync" for the PyTorch user self.precision_plugin.run_backward(tensor, *args, **kwargs) def optimizer_step( @@ -269,6 +267,7 @@ def optimizer_step( optimizer: the optimizer performing the step opt_idx: index of the current optimizer lambda_closure: closure calculating the loss value + model: reference to the model, optionally defining optimizer step related hooks """ model = model if model is not None else self.lightning_module make_optimizer_step = self.precision_plugin.pre_optimizer_step( From 0aaa43da2efd3dab87c7a8ae0ee0f4d3143dc1e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 13 Oct 2021 22:28:07 +0200 Subject: [PATCH 060/331] clean up examples --- pl_examples/lite_examples/gan/gan_example.py | 11 +++++------ pl_examples/lite_examples/simple/mnist_example.py | 8 ++++++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pl_examples/lite_examples/gan/gan_example.py b/pl_examples/lite_examples/gan/gan_example.py index c4a5e34d97d22..036593c09c314 100644 --- a/pl_examples/lite_examples/gan/gan_example.py +++ b/pl_examples/lite_examples/gan/gan_example.py @@ -22,6 +22,7 @@ import torchvision.utils as vutils from pl_examples.lite_examples.gan.models import Discriminator, Generator, weights_init +from pytorch_lightning import seed_everything from pytorch_lightning.lite import LightningLite from pytorch_lightning.lite.wrappers import _LiteOptimizer, _LiteModule @@ -53,8 +54,7 @@ class GANTrainer(LightningLite): def run(self): print("selected plugin: ", self._training_type_plugin) - random.seed(123) - torch.manual_seed(123) + seed_everything(123) # TODO: how do we handle this in Accelerator? # torch.cuda.set_device(opt.local_rank) @@ -62,9 +62,11 @@ def run(self): # os.environ["LOCAL_RANK"] = str(opt.local_rank) # os.environ["NODE_RANK"] = str(opt.local_rank) + if self.local_rank == 0: + dset.MNIST(root=".", download=True) + dataset = dset.MNIST( root=".", - download=True, transform=transforms.Compose( [ transforms.Resize(opt.imageSize), @@ -89,9 +91,6 @@ def run(self): self.to_device(netG) self.to_device(netD) - # assert isinstance(netG, DistributedDataParallel) - # assert isinstance(netD, DistributedDataParallel) - criterion = nn.BCELoss() fixed_noise = torch.randn(opt.batchSize, nz, 1, 1, device=self.device) diff --git a/pl_examples/lite_examples/simple/mnist_example.py b/pl_examples/lite_examples/simple/mnist_example.py index 396b3720538a5..ae093630ed8a7 100644 --- a/pl_examples/lite_examples/simple/mnist_example.py +++ b/pl_examples/lite_examples/simple/mnist_example.py @@ -6,6 +6,7 @@ from torchvision import datasets, transforms from torch.optim.lr_scheduler import StepLR +from pytorch_lightning import seed_everything from pytorch_lightning.lite import LightningLite @@ -39,7 +40,7 @@ class MNIST(LightningLite): def run(self, args): use_cuda = self.device.type == "cuda" - torch.manual_seed(args.seed) + seed_everything(args.seed) train_kwargs = {"batch_size": args.batch_size} test_kwargs = {"batch_size": args.test_batch_size} @@ -48,8 +49,11 @@ def run(self, args): train_kwargs.update(cuda_kwargs) test_kwargs.update(cuda_kwargs) + if self.local_rank == 0: + datasets.MNIST("../data", download=True) + transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]) - dataset1 = datasets.MNIST("../data", train=True, download=True, transform=transform) + dataset1 = datasets.MNIST("../data", train=True, transform=transform) dataset2 = datasets.MNIST("../data", train=False, transform=transform) train_loader = torch.utils.data.DataLoader(dataset1, **train_kwargs) test_loader = torch.utils.data.DataLoader(dataset2, **test_kwargs) From 52004b09b4c5c5ed6a244eefec6baf00e3485af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 14 Oct 2021 09:56:56 +0200 Subject: [PATCH 061/331] remove comment --- pytorch_lightning/plugins/precision/native_amp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pytorch_lightning/plugins/precision/native_amp.py b/pytorch_lightning/plugins/precision/native_amp.py index b5dbf3819f3e0..20b0d29b20f20 100644 --- a/pytorch_lightning/plugins/precision/native_amp.py +++ b/pytorch_lightning/plugins/precision/native_amp.py @@ -72,7 +72,6 @@ def pre_backward(self, model: "pl.LightningModule", closure_loss: torch.Tensor) def run_backward(self, tensor, *args, **kwargs): tensor = self.scaler.scale(tensor) super().run_backward(tensor, *args, **kwargs) - # self.scaler.unscale_(optimizer) # TODO: needed? def pre_optimizer_step( self, From c7ddd4059149d397a12eeffe0c1f83d561713b86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 15 Oct 2021 09:39:51 +0200 Subject: [PATCH 062/331] deepspeed --- .../simple/deepspeed_mnist_example.py | 155 ++++++++++++++++++ pytorch_lightning/accelerators/accelerator.py | 4 +- pytorch_lightning/lite/lite.py | 12 +- pytorch_lightning/lite/wrappers.py | 29 +++- .../plugins/precision/deepspeed_precision.py | 14 +- .../plugins/precision/native_amp.py | 4 +- .../plugins/precision/precision_plugin.py | 2 +- .../plugins/training_type/deepspeed.py | 45 ++++- .../training_type/training_type_plugin.py | 6 +- 9 files changed, 244 insertions(+), 27 deletions(-) create mode 100644 pl_examples/lite_examples/simple/deepspeed_mnist_example.py diff --git a/pl_examples/lite_examples/simple/deepspeed_mnist_example.py b/pl_examples/lite_examples/simple/deepspeed_mnist_example.py new file mode 100644 index 0000000000000..15b1fc036e9c6 --- /dev/null +++ b/pl_examples/lite_examples/simple/deepspeed_mnist_example.py @@ -0,0 +1,155 @@ +import argparse +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torchvision import datasets, transforms +from torch.optim.lr_scheduler import StepLR + +from pytorch_lightning import seed_everything +from pytorch_lightning.lite import LightningLite + + +class Net(nn.Module): + def __init__(self): + super(Net, self).__init__() + self.conv1 = nn.Conv2d(1, 32, 3, 1) + self.conv2 = nn.Conv2d(32, 64, 3, 1) + self.dropout1 = nn.Dropout(0.25) + self.dropout2 = nn.Dropout(0.5) + self.fc1 = nn.Linear(9216, 128) + self.fc2 = nn.Linear(128, 10) + + def forward(self, x): + x = self.conv1(x) + x = F.relu(x) + x = self.conv2(x) + x = F.relu(x) + x = F.max_pool2d(x, 2) + x = self.dropout1(x) + x = torch.flatten(x, 1) + x = self.fc1(x) + x = F.relu(x) + x = self.dropout2(x) + x = self.fc2(x) + output = F.log_softmax(x, dim=1) + return output + + +class DeepSpeedMNIST(LightningLite): + def run(self, args): + use_cuda = self.device.type == "cuda" + + seed_everything(args.seed) + + train_kwargs = {"batch_size": args.batch_size} + test_kwargs = {"batch_size": args.test_batch_size} + if use_cuda: + cuda_kwargs = {"num_workers": 1, "pin_memory": True, "shuffle": True} + train_kwargs.update(cuda_kwargs) + test_kwargs.update(cuda_kwargs) + + if self.local_rank == 0: + datasets.MNIST("../data", download=True) + + transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]) + dataset1 = datasets.MNIST("../data", train=True, transform=transform) + dataset2 = datasets.MNIST("../data", train=False, transform=transform) + train_loader = torch.utils.data.DataLoader(dataset1, **train_kwargs) + test_loader = torch.utils.data.DataLoader(dataset2, **test_kwargs) + + model = Net().to(self.device) + optimizer = optim.Adadelta(model.parameters(), lr=args.lr) + + train_loader, test_loader = self.setup_dataloader(train_loader, test_loader) + model, optimizer = self.setup(model, optimizer) + + scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) + for epoch in range(1, args.epochs + 1): + self.train(args, model, train_loader, optimizer, epoch) + self.test(model, test_loader) + scheduler.step() + + if args.save_model: + torch.save(model.state_dict(), "mnist_cnn.pt") + + def train(self, args, model, train_loader, optimizer, epoch): + model.train() + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(self.device), target.to(self.device) + # TODO: model.zero_grad() vs. optimizer.zero_grad() with deepspeed? + optimizer.zero_grad() + output = model(data) + loss = F.nll_loss(output, target) + # DEEPSPEED requires you to change loss.backward() to model.backward(loss) + model.backward(loss) + # DEEPSPEED requires you to change optimizer.step() to model.backward(loss) + model.step() + if batch_idx % args.log_interval == 0: + print( + "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format( + epoch, + batch_idx * len(data), + len(train_loader.dataset), + 100.0 * batch_idx / len(train_loader), + loss.item(), + ) + ) + if args.dry_run: + break + + def test(self, model, test_loader): + model.eval() + test_loss = 0 + correct = 0 + with torch.no_grad(): + for data, target in test_loader: + data, target = data.to(self.device), target.to(self.device) + output = model(data) + test_loss += F.nll_loss(output, target, reduction="sum").item() # sum up batch loss + pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability + correct += pred.eq(target.view_as(pred)).sum().item() + + test_loss /= len(test_loader.dataset) + + print( + "\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n".format( + test_loss, correct, len(test_loader.dataset), 100.0 * correct / len(test_loader.dataset) + ) + ) + + +def main(): + # Training settings + parser = argparse.ArgumentParser(description="PyTorch MNIST Example") + parser.add_argument( + "--batch-size", type=int, default=64, metavar="N", help="input batch size for training (default: 64)" + ) + parser.add_argument( + "--test-batch-size", type=int, default=1000, metavar="N", help="input batch size for testing (default: 1000)" + ) + parser.add_argument("--epochs", type=int, default=14, metavar="N", help="number of epochs to train (default: 14)") + parser.add_argument("--lr", type=float, default=1.0, metavar="LR", help="learning rate (default: 1.0)") + parser.add_argument("--gamma", type=float, default=0.7, metavar="M", help="Learning rate step gamma (default: 0.7)") + parser.add_argument("--dry-run", action="store_true", default=False, help="quickly check a single pass") + parser.add_argument("--seed", type=int, default=1, metavar="S", help="random seed (default: 1)") + parser.add_argument( + "--log-interval", + type=int, + default=10, + metavar="N", + help="how many batches to wait before logging training status", + ) + parser.add_argument("--save-model", action="store_true", default=False, help="For Saving the current Model") + parser.add_argument("--accelerator", type=str, default=None) + parser.add_argument("--gpus", type=int, default=None) + parser.add_argument("--num_processes", type=int, default=1) + parser.add_argument("--precision", type=int, default=32) + args = parser.parse_args() + + mnist = DeepSpeedMNIST(gpus=2, strategy="deepspeed", num_processes=1, precision=args.precision) + mnist.run(args) + + +if __name__ == "__main__": + main() diff --git a/pytorch_lightning/accelerators/accelerator.py b/pytorch_lightning/accelerators/accelerator.py index db033eae3f5bd..28cd05da195bd 100644 --- a/pytorch_lightning/accelerators/accelerator.py +++ b/pytorch_lightning/accelerators/accelerator.py @@ -249,9 +249,9 @@ def backward(self, closure_loss: Tensor, *args: Any, **kwargs: Any) -> Tensor: return closure_loss - def run_backward(self, tensor: Tensor, *args, **kwargs) -> None: + def run_backward(self, tensor: Tensor, model, *args, **kwargs) -> None: """Lightning-independent backward logic""" - self.precision_plugin.run_backward(tensor, *args, **kwargs) + self.precision_plugin.run_backward(tensor, model, *args, **kwargs) def optimizer_step( self, diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 6aa18390ba5e1..55320b6390a7a 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -145,22 +145,14 @@ def setup_dataloader(self, *dataloaders: DataLoader) -> Union[DataLoader, Sequen return dataloaders def backward(self, tensor: Tensor, *args: Any, **kwargs: Any) -> None: - # user will call automator.backward(loss) instead of loss.backward() - self._accelerator.run_backward(tensor, *args, **kwargs) + # user will call self.backward(loss) instead of loss.backward() + self._accelerator.run_backward(tensor, *args, model=None, **kwargs) @contextmanager def forward_context(self) -> Generator[None, None, None]: with self._accelerator.forward_context(): yield - # @contextmanager - # def optimizer_step_context(self, model=None, optimizer=None): - # # necessary for deepspeed + scaling - # temp = optimizer.step - # optimizer.step = model.step - # yield - # optimizer.step = temp - def to_device(self, obj: Union[nn.Module, Tensor, Any]) -> Union[nn.Module, Tensor, Any]: if isinstance(obj, nn.Module): return obj.to(self.device) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 19bb8d34c44c1..1f7e7d33858d3 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -18,17 +18,26 @@ from torch.optim import Optimizer from pytorch_lightning.accelerators import Accelerator +from pytorch_lightning.plugins import DeepSpeedPlugin from pytorch_lightning.utilities.apply_func import apply_to_collection +# TODO: add attributes and methods from Optimizer class _LiteOptimizer(Optimizer): def __init__(self, optimizer: Optimizer, accelerator: Accelerator) -> None: - super().__init__(params=optimizer.param_groups, defaults=optimizer.defaults) # type: ignore[call-arg] - self.__dict__ = optimizer.__dict__ + super().__init__(params=optimizer.param_groups, defaults=getattr(optimizer, "defaults", {})) # type: ignore[call-arg] self._optimizer = optimizer self._accelerator = accelerator + @property + def optimizer(self) -> Optimizer: + return self._optimizer + def step(self, closure: Optional[Callable] = None) -> None: + if isinstance(self._accelerator.training_type_plugin, DeepSpeedPlugin): + self._optimizer.step(closure) + return + self._accelerator.optimizer_step( self._optimizer, lambda_closure=closure, @@ -52,3 +61,19 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: output = apply_to_collection(output, function=lambda t: t.to(torch.get_default_dtype()), dtype=Tensor) return output + + def backward(self, loss, *args, **kwargs): + if not isinstance(self._accelerator.training_type_plugin, DeepSpeedPlugin): + raise RuntimeError( + f"Calling `.backward()` on {self.module.__class__.__name__} is not allowed." + f" Please change your code to call `backward()` on the loss tensor directly." + ) + self._accelerator.run_backward(loss, self.module, *args, **kwargs) + + def step(self): + if not isinstance(self._accelerator.training_type_plugin, DeepSpeedPlugin): + raise RuntimeError( + f"Calling `.step()` on {self.module.__class__.__name__} is not allowed." + f" Please change your code to call the optimizer's `step()` method instead." + ) + self.module.step() diff --git a/pytorch_lightning/plugins/precision/deepspeed_precision.py b/pytorch_lightning/plugins/precision/deepspeed_precision.py index 03be19e143c72..774fbe2a74ab1 100644 --- a/pytorch_lightning/plugins/precision/deepspeed_precision.py +++ b/pytorch_lightning/plugins/precision/deepspeed_precision.py @@ -13,6 +13,7 @@ # limitations under the License. from typing import Any, Callable, Optional, Union +import deepspeed from torch import Tensor from torch.nn import Module from torch.optim import Optimizer @@ -36,22 +37,22 @@ def __init__(self, precision: int) -> None: def pre_optimizer_step( self, - model: "pl.LightningModule", + model: Union[Module, "pl.LightningModule"], optimizer: Optimizer, optimizer_idx: int, lambda_closure: Callable, **kwargs: Any, ) -> bool: """Hook to do something before each optimizer step.""" - result = lambda_closure() # DeepSpeed does not support closures + result = lambda_closure() if lambda_closure is not None else None # DeepSpeed does not support closures super().pre_optimizer_step(model, optimizer, optimizer_idx, lambda_closure, **kwargs) # in manual optimization, the closure does not return a value - if model.automatic_optimization and result is None: + if isinstance(model, pl.LightningModule) and model.automatic_optimization and result is None: raise MisconfigurationException( "Skipping backward by returning `None` from your `training_step` is not supported by `DeepSpeed`" ) # the following should be in a `optimizer_step` hook but we don't have one in the precision plugin. - deepspeed_engine = model.trainer.model + deepspeed_engine = model.trainer.model if isinstance(model, pl.LightningModule) else model deepspeed_engine.step() return False @@ -65,9 +66,8 @@ def backward(self, model: "pl.LightningModule", closure_loss: Tensor, *args: Any deepspeed_engine = model.trainer.model deepspeed_engine.backward(closure_loss, *args, **kwargs) - def run_backward(self, tensor, *args, **kwargs): - deepspeed_engine = None # TODO: access engine here - deepspeed_engine.backward(tensor, *args, **kwargs) + def run_backward(self, tensor, model, *args, **kwargs): + model.backward(tensor, *args, **kwargs) def clip_gradients( self, diff --git a/pytorch_lightning/plugins/precision/native_amp.py b/pytorch_lightning/plugins/precision/native_amp.py index 20b0d29b20f20..27f0fcbccf4c5 100644 --- a/pytorch_lightning/plugins/precision/native_amp.py +++ b/pytorch_lightning/plugins/precision/native_amp.py @@ -69,9 +69,9 @@ def pre_backward(self, model: "pl.LightningModule", closure_loss: torch.Tensor) closure_loss = self.scaler.scale(closure_loss) return super().pre_backward(model, closure_loss) - def run_backward(self, tensor, *args, **kwargs): + def run_backward(self, tensor, model, *args, **kwargs): tensor = self.scaler.scale(tensor) - super().run_backward(tensor, *args, **kwargs) + super().run_backward(tensor, model, *args, **kwargs) def pre_optimizer_step( self, diff --git a/pytorch_lightning/plugins/precision/precision_plugin.py b/pytorch_lightning/plugins/precision/precision_plugin.py index f8b56d1f746ee..e06f7ac7ccc99 100644 --- a/pytorch_lightning/plugins/precision/precision_plugin.py +++ b/pytorch_lightning/plugins/precision/precision_plugin.py @@ -90,7 +90,7 @@ def post_backward(self, model: "pl.LightningModule", closure_loss: Tensor) -> Te model.trainer.call_hook("on_after_backward") return closure_loss - def run_backward(self, tensor, *args, **kwargs) -> None: + def run_backward(self, tensor, model, *args, **kwargs) -> None: """Lightning-independent backward logic.""" tensor.backward(*args, **kwargs) diff --git a/pytorch_lightning/plugins/training_type/deepspeed.py b/pytorch_lightning/plugins/training_type/deepspeed.py index e2e8c316f48d1..764bcfab18b87 100644 --- a/pytorch_lightning/plugins/training_type/deepspeed.py +++ b/pytorch_lightning/plugins/training_type/deepspeed.py @@ -19,9 +19,11 @@ import platform from collections import OrderedDict from pathlib import Path -from typing import Any, Callable, Dict, Generator, List, Mapping, Optional, Tuple, Union +from typing import Any, Callable, Dict, Generator, List, Mapping, Optional, Tuple, Union, Sequence import torch +from deepspeed import DeepSpeedEngine +from torch.nn import Module from torch.optim import Optimizer import pytorch_lightning as pl @@ -337,6 +339,7 @@ def _load_config(self, config): config = json.load(f) return config + # getting called by Lightning trainer AND Lite def setup_distributed(self): reset_seed() @@ -377,6 +380,41 @@ def pre_dispatch(self): self.init_deepspeed() self.barrier() + # TODO: avoid code duplication by letting the plugin reuse this method + def setup_models_and_optimizers( + self, models: Sequence[Module], optimizers: Sequence[Optimizer] + ) -> Tuple[Sequence[Module], Sequence[Optimizer]]: + if len(models) != len(optimizers): + raise ValueError( + f"DeepSpeed requires one optimizer per model." + f" Got {len(models)} models and {len(optimizers)} optimizers instead." + ) + + # TODO: is this the correct place to set this? + self.config["train_micro_batch_size_per_gpu"] = 1 + + models_and_optimizers = [ + self._setup_model_and_optimizer(model, optimizer) for model, optimizer in zip(models, optimizers) + ] + models, optimizers = zip(*models_and_optimizers) + + # TODO: do we need to call it here? + # self._set_deepspeed_activation_checkpointing() + return list(models), list(optimizers) + + def _setup_model_and_optimizer(self, model: Module, optimizer: Optimizer) -> Tuple[DeepSpeedEngine, Optimizer]: + # TODO: shouldn't this be optimizer.parameters? + model_parameters = filter(lambda p: p.requires_grad, model.parameters()) + deepspeed_engine, deepspeed_optimizer, _, _ = deepspeed.initialize( + args=argparse.Namespace(device_rank=self.root_device.index), + config=self.config, + model=model, + model_parameters=model_parameters, # TODO: is the type correct here? + optimizer=optimizer, + dist_init_required=False, + ) + return deepspeed_engine, deepspeed_optimizer + def init_deepspeed(self): # check that `configure_gradient_clipping` hook isn't overriden since deepspeed handles # gradient clipping internally @@ -568,6 +606,9 @@ def _format_config(self): self._format_precision_config() def _format_batch_size_and_grad_accum_config(self): + if self.lightning_module is None: + return + if "gradient_accumulation_steps" in self.config: raise MisconfigurationException( "Do not set `gradient_accumulation_steps` in the DeepSpeed config" @@ -596,6 +637,8 @@ def _auto_select_batch_size(self): return batch_size def _format_precision_config(self): + # TODO: support precision + return amp_type = self.lightning_module.trainer.accelerator_connector.amp_type amp_level = self.lightning_module.trainer.accelerator_connector.amp_level precision = self.lightning_module.trainer.accelerator_connector.precision diff --git a/pytorch_lightning/plugins/training_type/training_type_plugin.py b/pytorch_lightning/plugins/training_type/training_type_plugin.py index 4f6d4c24b6668..bc0dd7218e176 100644 --- a/pytorch_lightning/plugins/training_type/training_type_plugin.py +++ b/pytorch_lightning/plugins/training_type/training_type_plugin.py @@ -13,7 +13,7 @@ # limitations under the License. import contextlib from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, Generator, Iterable, Mapping, Optional, Sequence, Union +from typing import Any, Callable, Dict, Generator, Iterable, Mapping, Optional, Sequence, Union, Tuple import torch from torch import Tensor @@ -65,7 +65,9 @@ def setup_dataloader(self, dataloader: DataLoader) -> DataLoader: """Called by the accelerator. The plugin wraps and modifies the dataloader as needed.""" return dataloader - def setup_models_and_optimizers(self, models: Sequence[Module], optimizers: Sequence[Optimizer]): + def setup_models_and_optimizers( + self, models: Sequence[Module], optimizers: Sequence[Optimizer] + ) -> Tuple[Sequence[Module], Sequence[Optimizer]]: models = [self.setup_model(model) for model in models] optimizers = [self.setup_optimizer(optimizer) for optimizer in optimizers] return models, optimizers From a3879d11821d8e66ddd8619ba56d9217316d03aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 15 Oct 2021 09:44:46 +0200 Subject: [PATCH 063/331] deepspeed gan exmple --- .../gan/deep_speed_gan_example.py | 176 ++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 pl_examples/lite_examples/gan/deep_speed_gan_example.py diff --git a/pl_examples/lite_examples/gan/deep_speed_gan_example.py b/pl_examples/lite_examples/gan/deep_speed_gan_example.py new file mode 100644 index 0000000000000..e5e7e2c22ab06 --- /dev/null +++ b/pl_examples/lite_examples/gan/deep_speed_gan_example.py @@ -0,0 +1,176 @@ +""" +DCGAN - Adapted from pytorch/examples + +Launch it with this command: + +python -m torch.distributed.run --nproc_per_node=2 gan_example.py + +""" +from __future__ import print_function + +import argparse +import os +import random + +import torch +import torch.nn as nn +import torch.nn.parallel +import torch.optim as optim +import torch.utils.data +import torchvision.datasets as dset +import torchvision.transforms as transforms +import torchvision.utils as vutils + +from pl_examples.lite_examples.gan.models import Discriminator, Generator, weights_init +from pytorch_lightning import seed_everything +from pytorch_lightning.lite import LightningLite +from pytorch_lightning.lite.wrappers import _LiteOptimizer, _LiteModule + +parser = argparse.ArgumentParser() +parser.add_argument("--workers", type=int, help="number of data loading workers", default=0) +parser.add_argument("--batchSize", type=int, default=64, help="input batch size") +parser.add_argument( + "--imageSize", + type=int, + default=64, + help="the height / width of the input image to network", +) +parser.add_argument("--niter", type=int, default=25, help="number of epochs to train for") +parser.add_argument("--lr", type=float, default=0.0002, help="learning rate, default=0.0002") +parser.add_argument("--beta1", type=float, default=0.5, help="beta1 for adam. default=0.5") +parser.add_argument("--ngpu", type=int, default=1, help="number of GPUs to use") +parser.add_argument("--netG", default="", help="path to netG (to continue training)") +parser.add_argument("--netD", default="", help="path to netD (to continue training)") +parser.add_argument("--outf", default="./lightning_logs", help="folder to output images and model checkpoints") +parser.add_argument("--local_rank", type=int, default=0) + +opt, _ = parser.parse_known_args() +os.makedirs(opt.outf, exist_ok=True) +ngpu = int(opt.ngpu) + +nz = 100 + + +class GANTrainer(LightningLite): + def run(self): + print("selected plugin: ", self._training_type_plugin) + seed_everything(123) + + # TODO: how do we handle this in Accelerator? + # torch.cuda.set_device(opt.local_rank) + # TODO: how do we handle this? + # os.environ["LOCAL_RANK"] = str(opt.local_rank) + # os.environ["NODE_RANK"] = str(opt.local_rank) + + if self.local_rank == 0: + dset.MNIST(root=".", download=True) + + dataset = dset.MNIST( + root=".", + transform=transforms.Compose( + [ + transforms.Resize(opt.imageSize), + transforms.ToTensor(), + transforms.Normalize((0.5,), (0.5,)), + ] + ), + ) + dataloader = torch.utils.data.DataLoader( + dataset, batch_size=opt.batchSize, shuffle=True, num_workers=opt.workers + ) + + dataloader = self.setup_dataloader(dataloader) + # assert isinstance(dataloader.sampler, DistributedSampler) + + netG = Generator() + netG.apply(weights_init) + + netD = Discriminator() + netD.apply(weights_init) + + self.to_device(netG) + self.to_device(netD) + + criterion = nn.BCELoss() + + fixed_noise = torch.randn(opt.batchSize, nz, 1, 1, device=self.device) + real_label = 1 + fake_label = 0 + + # setup optimizer + optimizerD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) + optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) + + (netG, netD), (optimizerG, optimizerD) = self.setup(models=(netG, netD), optimizers=(optimizerG, optimizerD)) + + assert isinstance(optimizerG, _LiteOptimizer) + assert isinstance(netG, _LiteModule) + + for epoch in range(opt.niter): + for i, data in enumerate(dataloader, 0): + ############################ + # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z))) + ########################### + # train with real + netD.zero_grad() + real_cpu = self.to_device(data[0]) + batch_size = real_cpu.size(0) + label = torch.full((batch_size,), real_label, dtype=real_cpu.dtype, device=self.device) + + output = netD(real_cpu) + errD_real = criterion(output, label) + netD.backward(errD_real) + D_x = output.mean().item() + + # train with fake + noise = torch.randn(batch_size, nz, 1, 1, device=self.device) + fake = netG(noise) + label.fill_(fake_label) + output = netD(fake.detach()) + errD_fake = criterion(output, label) + netD.backward(errD_fake) + D_G_z1 = output.mean().item() + errD = errD_real + errD_fake + netD.step() + + ############################ + # (2) Update G network: maximize log(D(G(z))) + ########################### + netG.zero_grad() + label.fill_(real_label) # fake labels are real for generator cost + output = netD(fake) + errG = criterion(output, label) + netG.backward(errG) + D_G_z2 = output.mean().item() + netG.step() + + print( + "[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)): %.4f / %.4f" + % ( + epoch, + opt.niter, + i, + len(dataloader), + errD.item(), + errG.item(), + D_x, + D_G_z1, + D_G_z2, + ) + ) + if i % 100 == 0: + vutils.save_image(real_cpu, "%s/real_samples.png" % opt.outf, normalize=True) + fake = netG(fixed_noise) + vutils.save_image( + fake.detach(), + "%s/fake_samples_epoch_%03d.png" % (opt.outf, epoch), + normalize=True, + ) + # do checkpointing + # torch.save(netG.state_dict(), "%s/netG_epoch_%d.pth" % (opt.outf, epoch)) + # torch.save(netD.state_dict(), "%s/netD_epoch_%d.pth" % (opt.outf, epoch)) + + +if __name__ == "__main__": + gan = GANTrainer(gpus=2, accelerator="deepspeed") + gan.run() From 66c9addaa9542e7508b48bc04c315576806e7d3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 15 Oct 2021 09:49:48 +0200 Subject: [PATCH 064/331] x --- pl_examples/lite_examples/gan/deep_speed_gan_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/lite_examples/gan/deep_speed_gan_example.py b/pl_examples/lite_examples/gan/deep_speed_gan_example.py index e5e7e2c22ab06..2cd6739562cb9 100644 --- a/pl_examples/lite_examples/gan/deep_speed_gan_example.py +++ b/pl_examples/lite_examples/gan/deep_speed_gan_example.py @@ -172,5 +172,5 @@ def run(self): if __name__ == "__main__": - gan = GANTrainer(gpus=2, accelerator="deepspeed") + gan = GANTrainer(gpus=1, accelerator="deepspeed") gan.run() From f2080171dd4b1c2b98415473180f998c1bd9d0e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 15 Oct 2021 09:50:34 +0200 Subject: [PATCH 065/331] rename --- .../gan/{deep_speed_gan_example.py => deepspeed_gan_example.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pl_examples/lite_examples/gan/{deep_speed_gan_example.py => deepspeed_gan_example.py} (100%) diff --git a/pl_examples/lite_examples/gan/deep_speed_gan_example.py b/pl_examples/lite_examples/gan/deepspeed_gan_example.py similarity index 100% rename from pl_examples/lite_examples/gan/deep_speed_gan_example.py rename to pl_examples/lite_examples/gan/deepspeed_gan_example.py From 54f9cdd430b157b86e24a189d1a86abc258f5cf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 15 Oct 2021 10:04:47 +0200 Subject: [PATCH 066/331] move protected to bottom --- pytorch_lightning/lite/lite.py | 44 +++++++++++++++++----------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 55320b6390a7a..db4334db4772c 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -102,17 +102,6 @@ def world_size(self) -> int: def run(self, *args: Any, **kwargs: Any) -> None: pass - def _run_wrapper(self, run_method: Callable) -> Callable: - return partial(self._run_impl, run_method) - - def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> None: - self._training_type_plugin.setup_environment() - if isinstance(self._training_type_plugin, DDPSpawnPlugin): - self._training_type_plugin.spawn(run_method, *args, **kwargs) - else: - run_method(*args, **kwargs) - # TODO: any teardown needed here? - def setup( self, models: Union[nn.Module, Sequence[nn.Module]], @@ -127,17 +116,6 @@ def setup( optimizers = optimizers[0] if len(optimizers) == 1 else optimizers return models, optimizers - def _setup_models_and_optimizers( - self, - models: Sequence[nn.Module], - optimizers: Sequence[Optimizer], - ) -> Tuple[Sequence[_LiteModule], Sequence[_LiteOptimizer]]: - # Let accelerator/plugin wrap and connect the models and optimizers - models, optimizers = self._training_type_plugin.setup_models_and_optimizers(models, optimizers) - models = [_LiteModule(module=model, accelerator=self._accelerator) for model in models] - optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self._accelerator) for optimizer in optimizers] - return models, optimizers - def setup_dataloader(self, *dataloaders: DataLoader) -> Union[DataLoader, Sequence[DataLoader]]: # user can call this method independently instead of the general purpose setup method dataloaders = [self._training_type_plugin.setup_dataloader(dataloader) for dataloader in dataloaders] @@ -167,3 +145,25 @@ def save_checkpoint(self, filepath: Union[str, Path], content: Dict[str, Any]) - def execute_on_rank(self, func: Callable, rank: int, *args: Any, **kwargs: Any) -> None: if self.global_rank == rank: func(*args, **kwargs) + + def _run_wrapper(self, run_method: Callable) -> Callable: + return partial(self._run_impl, run_method) + + def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> None: + self._training_type_plugin.setup_environment() + if isinstance(self._training_type_plugin, DDPSpawnPlugin): + self._training_type_plugin.spawn(run_method, *args, **kwargs) + else: + run_method(*args, **kwargs) + # TODO: any teardown needed here? + + def _setup_models_and_optimizers( + self, + models: Sequence[nn.Module], + optimizers: Sequence[Optimizer], + ) -> Tuple[Sequence[_LiteModule], Sequence[_LiteOptimizer]]: + # Let accelerator/plugin wrap and connect the models and optimizers + models, optimizers = self._training_type_plugin.setup_models_and_optimizers(models, optimizers) + models = [_LiteModule(module=model, accelerator=self._accelerator) for model in models] + optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self._accelerator) for optimizer in optimizers] + return models, optimizers From c8934fe855c17fb79480db206445c31f77618598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 15 Oct 2021 10:32:31 +0200 Subject: [PATCH 067/331] distributed sampler setup --- pl_examples/lite_examples/gan/gan_example.py | 1 + .../lite_examples/simple/mnist_example.py | 5 +- pytorch_lightning/lite/lite.py | 51 ++++++++++++++++--- 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/pl_examples/lite_examples/gan/gan_example.py b/pl_examples/lite_examples/gan/gan_example.py index 036593c09c314..c4c09feb52522 100644 --- a/pl_examples/lite_examples/gan/gan_example.py +++ b/pl_examples/lite_examples/gan/gan_example.py @@ -20,6 +20,7 @@ import torchvision.datasets as dset import torchvision.transforms as transforms import torchvision.utils as vutils +from torch.utils.data import DistributedSampler from pl_examples.lite_examples.gan.models import Discriminator, Generator, weights_init from pytorch_lightning import seed_everything diff --git a/pl_examples/lite_examples/simple/mnist_example.py b/pl_examples/lite_examples/simple/mnist_example.py index ae093630ed8a7..934fb0743b7f7 100644 --- a/pl_examples/lite_examples/simple/mnist_example.py +++ b/pl_examples/lite_examples/simple/mnist_example.py @@ -3,6 +3,7 @@ import torch.nn as nn import torch.nn.functional as F import torch.optim as optim +from torch.utils.data import DistributedSampler from torchvision import datasets, transforms from torch.optim.lr_scheduler import StepLR @@ -62,6 +63,8 @@ def run(self, args): optimizer = optim.Adadelta(model.parameters(), lr=args.lr) train_loader, test_loader = self.setup_dataloader(train_loader, test_loader) + assert isinstance(train_loader.sampler, DistributedSampler) + assert isinstance(test_loader.sampler, DistributedSampler) model, optimizer = self.setup(model, optimizer) scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) @@ -100,7 +103,7 @@ def test(self, model, test_loader): test_loss = 0 correct = 0 with torch.no_grad(): - for data, target in test_loader: + for i, (data, target) in enumerate(test_loader): data, target = data.to(self.device), target.to(self.device) output = model(data) test_loss += F.nll_loss(output, target, reduction="sum").item() # sum up batch loss diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index db4334db4772c..619d7068d26cd 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os from abc import abstractmethod, ABC from collections import Callable from contextlib import contextmanager @@ -23,14 +23,17 @@ import torch.nn as nn from torch import Tensor from torch.optim import Optimizer -from torch.utils.data import DataLoader +from torch.utils.data import DataLoader, DistributedSampler, SequentialSampler, RandomSampler, Sampler from pytorch_lightning import Trainer -from pytorch_lightning.accelerators import Accelerator +from pytorch_lightning.accelerators import Accelerator, TPUAccelerator from pytorch_lightning.lite.wrappers import _LiteOptimizer, _LiteModule from pytorch_lightning.plugins import PLUGIN_INPUT, DDPSpawnPlugin, TrainingTypePlugin from pytorch_lightning.trainer.connectors.accelerator_connector import AcceleratorConnector +from pytorch_lightning.trainer.data_loading import TrainerDataLoadingMixin from pytorch_lightning.utilities import move_data_to_device +from pytorch_lightning.utilities.data import has_iterable_dataset +from pytorch_lightning.utilities.exceptions import MisconfigurationException class LightningLite(ABC): @@ -51,7 +54,7 @@ def __init__( replace_sampler_ddp: bool = True, ) -> None: gpu_ids, tpu_cores = Trainer._parse_devices(gpus=gpus, auto_select_gpus=False, tpu_cores=tpu_cores) - backend_connector = AcceleratorConnector( + self._accelerator_connector = AcceleratorConnector( num_processes=num_processes, devices=devices, tpu_cores=tpu_cores, @@ -71,7 +74,7 @@ def __init__( amp_level=amp_level, plugins=plugins, ) - self._accelerator = backend_connector.select_accelerator() + self._accelerator = self._accelerator_connector.select_accelerator() self._training_type_plugin = self._accelerator.training_type_plugin self._precision_plugin = self._accelerator.precision_plugin @@ -118,7 +121,12 @@ def setup( def setup_dataloader(self, *dataloaders: DataLoader) -> Union[DataLoader, Sequence[DataLoader]]: # user can call this method independently instead of the general purpose setup method - dataloaders = [self._training_type_plugin.setup_dataloader(dataloader) for dataloader in dataloaders] + # dataloaders = [self._training_type_plugin.setup_dataloader(dataloader) for dataloader in dataloaders] + dataloaders = [ + TrainerDataLoadingMixin._update_dataloader(dataloader, sampler=self._resolve_sampler(dataloader)) + for dataloader in dataloaders + if self._requires_distributed_sampler(dataloader) or isinstance(self._accelerator, TPUAccelerator) + ] dataloaders = dataloaders[0] if len(dataloaders) == 1 else dataloaders return dataloaders @@ -167,3 +175,34 @@ def _setup_models_and_optimizers( models = [_LiteModule(module=model, accelerator=self._accelerator) for model in models] optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self._accelerator) for optimizer in optimizers] return models, optimizers + + # TODO: copied from data_loading.py + def _requires_distributed_sampler(self, dataloader: DataLoader) -> bool: + return ( + self._accelerator_connector.replace_sampler_ddp + and self._accelerator_connector.is_distributed + and not isinstance(dataloader.sampler, DistributedSampler) + and not has_iterable_dataset(dataloader) + ) + + # TODO: copied and adapted from data_loading.py + def _resolve_sampler(self, dataloader: DataLoader) -> Sampler: + if self._requires_distributed_sampler(dataloader): + if not isinstance(dataloader.sampler, (SequentialSampler, RandomSampler)): + raise MisconfigurationException( + "You seem to have configured a sampler in your DataLoader. This will be replaced " + " by `DistributedSampler` since `replace_sampler_ddp` is True and you are using" + " distributed training. Either remove the sampler from your DataLoader or set" + " `replace_sampler_ddp=False` if you want to use your custom sampler." + ) + return self._get_distributed_sampler(dataloader, **self._training_type_plugin.distributed_sampler_kwargs) + + return dataloader.sampler + + # TODO: copied and adapted from data_loading.py + @staticmethod + def _get_distributed_sampler(dataloader: DataLoader, **kwargs: Any) -> DistributedSampler: + """This function is used to created the distributed sampler injected within the user DataLoader.""" + kwargs.setdefault("seed", int(os.getenv("PL_GLOBAL_SEED", 0))) + sampler = DistributedSampler(dataloader.dataset, **kwargs) + return sampler From d10c3cfc52e7ff043644c86c0b0ad33118997e11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 15 Oct 2021 11:06:40 +0200 Subject: [PATCH 068/331] typing --- pytorch_lightning/lite/lite.py | 2 +- pytorch_lightning/lite/wrappers.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 619d7068d26cd..fc5d916f3818a 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -132,7 +132,7 @@ def setup_dataloader(self, *dataloaders: DataLoader) -> Union[DataLoader, Sequen def backward(self, tensor: Tensor, *args: Any, **kwargs: Any) -> None: # user will call self.backward(loss) instead of loss.backward() - self._accelerator.run_backward(tensor, *args, model=None, **kwargs) + self._accelerator.run_backward(tensor, None, *args, **kwargs) @contextmanager def forward_context(self) -> Generator[None, None, None]: diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 1f7e7d33858d3..3e1dbbd818c1d 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -62,7 +62,7 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: output = apply_to_collection(output, function=lambda t: t.to(torch.get_default_dtype()), dtype=Tensor) return output - def backward(self, loss, *args, **kwargs): + def backward(self, loss, *args: Any, **kwargs: Any) -> None: if not isinstance(self._accelerator.training_type_plugin, DeepSpeedPlugin): raise RuntimeError( f"Calling `.backward()` on {self.module.__class__.__name__} is not allowed." @@ -70,7 +70,7 @@ def backward(self, loss, *args, **kwargs): ) self._accelerator.run_backward(loss, self.module, *args, **kwargs) - def step(self): + def step(self) -> None: if not isinstance(self._accelerator.training_type_plugin, DeepSpeedPlugin): raise RuntimeError( f"Calling `.step()` on {self.module.__class__.__name__} is not allowed." From eae03b56b01448c9ed4647854e81cd568936c712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 15 Oct 2021 11:06:50 +0200 Subject: [PATCH 069/331] self.print() --- pytorch_lightning/lite/lite.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index fc5d916f3818a..ac02fcb9feb99 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -144,6 +144,10 @@ def to_device(self, obj: Union[nn.Module, Tensor, Any]) -> Union[nn.Module, Tens return obj.to(self.device) return move_data_to_device(obj, device=self.device) + def print(self, *args: Any, **kwargs: Any) -> None: + if self.local_rank == 0: + print(*args, **kwargs) + def reduce_decision(self, decision: bool) -> bool: return self._training_type_plugin.reduce_boolean_decision(decision) From 633e02d38ddcfbaa2ece96e7ee88cdc74b1b1372 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 15 Oct 2021 13:04:02 +0200 Subject: [PATCH 070/331] sharded --- .../plugins/training_type/sharded.py | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/pytorch_lightning/plugins/training_type/sharded.py b/pytorch_lightning/plugins/training_type/sharded.py index d684a34784f4c..615d930e86e49 100644 --- a/pytorch_lightning/plugins/training_type/sharded.py +++ b/pytorch_lightning/plugins/training_type/sharded.py @@ -12,9 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. from contextlib import contextmanager -from typing import Dict, Generator, Optional +from typing import Dict, Generator, Optional, Sequence, Tuple, List, Union import torch +from torch import Module +from torch.optim import Optimizer import pytorch_lightning as pl from pytorch_lightning.core.optimizer import LightningOptimizer @@ -35,22 +37,33 @@ class DDPShardedPlugin(DDPPlugin): _REDUCE_BUFFER_SIZE_DEFAULT = 2 ** 23 # 8M - def configure_ddp(self) -> None: - self._wrap_optimizers() + def setup_models_and_optimizers( + self, models: Sequence[Module], optimizers: Sequence[Optimizer] + ) -> Tuple[Sequence[Module], Sequence[Optimizer]]: + if len(models) > 1: + raise ValueError( + f"DDPSharded only supports a single model with one or several optimizers. Got {len(models)} models." + ) + + optimizers = self._wrap_optimizers(optimizers) + model = ShardedDataParallel(models[0], sharded_optimizer=optimizers, **self._ddp_kwargs) + setattr(model, "require_backward_grad_sync", False) # TODO: needed? + return [model], optimizers + def configure_ddp(self) -> None: if "reduce_buffer_size" not in self._ddp_kwargs: # For multi-node training, enabling bucketing will improve performance. self._ddp_kwargs["reduce_buffer_size"] = self._REDUCE_BUFFER_SIZE_DEFAULT if self.num_nodes > 1 else 0 - self._model = ShardedDataParallel( - LightningShardedDataParallel(self.model), - sharded_optimizer=self.lightning_module.trainer.optimizers, - **self._ddp_kwargs + [self._model], optimizers = self.setup_models_and_optimizers( + models=[LightningShardedDataParallel(self.model)], + optimizers=self.lightning_module.trainer.optimizers, ) - setattr(self._model, "require_backward_grad_sync", False) + trainer = self.lightning_module.trainer + trainer.optimizers = optimizers + trainer.convert_to_lightning_optimizers() - def _reinit_optimizers_with_oss(self): - optimizers = self.lightning_module.trainer.optimizers + def _reinit_optimizers_with_oss(self, optimizers: Sequence[Union[Optimizer, LightningOptimizer]]) -> Sequence[OSS]: for x, optimizer in enumerate(optimizers): if isinstance(optimizer, LightningOptimizer): optimizer = optimizer._optimizer @@ -58,7 +71,7 @@ def _reinit_optimizers_with_oss(self): optim_class = type(optimizer) zero_optimizer = OSS(params=optimizer.param_groups, optim=optim_class, **optimizer.defaults) if _FAIRSCALE_OSS_FP16_BROADCAST_AVAILABLE: - precision = self.lightning_module.trainer.precision + precision = self.lightning_module.trainer.precision # TODO: how to handle this?! is_fp16 = precision in ("mixed", 16) # For multi-node training, compressing the model shards in fp16 before broadcasting # improves performance. When using PyTorch AMP, it will not degrade @@ -66,14 +79,13 @@ def _reinit_optimizers_with_oss(self): zero_optimizer.broadcast_fp16 = is_fp16 and self.num_nodes > 1 optimizers[x] = zero_optimizer del optimizer - trainer = self.lightning_module.trainer - trainer.optimizers = optimizers - trainer.convert_to_lightning_optimizers() + return optimizers + + def _wrap_optimizers(self, optimizers: Sequence[Optimizer]) -> Sequence[OSS]: + if self.model is not None and self.model.trainer.state.fn != TrainerFn.FITTING: + return optimizers - def _wrap_optimizers(self): - if self.model.trainer.state.fn != TrainerFn.FITTING: - return - self._reinit_optimizers_with_oss() + return self._reinit_optimizers_with_oss(optimizers) def optimizer_state(self, optimizer: "OSS") -> Optional[dict]: if isinstance(optimizer, LightningOptimizer): From 40f28e21c3b5c43f0a58287b7965ea50551e9e64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 15 Oct 2021 13:05:47 +0200 Subject: [PATCH 071/331] x --- pytorch_lightning/plugins/training_type/sharded.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/plugins/training_type/sharded.py b/pytorch_lightning/plugins/training_type/sharded.py index 615d930e86e49..e1370e3691386 100644 --- a/pytorch_lightning/plugins/training_type/sharded.py +++ b/pytorch_lightning/plugins/training_type/sharded.py @@ -15,7 +15,7 @@ from typing import Dict, Generator, Optional, Sequence, Tuple, List, Union import torch -from torch import Module +from torch.nn import Module from torch.optim import Optimizer import pytorch_lightning as pl From b7dcc432091752b62abad0795cf631add7883ca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 15 Oct 2021 13:13:45 +0200 Subject: [PATCH 072/331] hack --- pytorch_lightning/plugins/training_type/sharded.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/plugins/training_type/sharded.py b/pytorch_lightning/plugins/training_type/sharded.py index e1370e3691386..34021ba15a2b7 100644 --- a/pytorch_lightning/plugins/training_type/sharded.py +++ b/pytorch_lightning/plugins/training_type/sharded.py @@ -71,7 +71,9 @@ def _reinit_optimizers_with_oss(self, optimizers: Sequence[Union[Optimizer, Ligh optim_class = type(optimizer) zero_optimizer = OSS(params=optimizer.param_groups, optim=optim_class, **optimizer.defaults) if _FAIRSCALE_OSS_FP16_BROADCAST_AVAILABLE: - precision = self.lightning_module.trainer.precision # TODO: how to handle this?! + precision = ( + 32 if self.lightning_module is None else self.lightning_module.trainer.precision + ) # TODO: how to handle this?! is_fp16 = precision in ("mixed", 16) # For multi-node training, compressing the model shards in fp16 before broadcasting # improves performance. When using PyTorch AMP, it will not degrade From d6e9f235f58c663060e660ef5e8eca5f33850dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 15 Oct 2021 13:32:38 +0200 Subject: [PATCH 073/331] x --- pl_examples/lite_examples/simple/mnist_example.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pl_examples/lite_examples/simple/mnist_example.py b/pl_examples/lite_examples/simple/mnist_example.py index 934fb0743b7f7..5849b2d86ee77 100644 --- a/pl_examples/lite_examples/simple/mnist_example.py +++ b/pl_examples/lite_examples/simple/mnist_example.py @@ -67,6 +67,8 @@ def run(self, args): assert isinstance(test_loader.sampler, DistributedSampler) model, optimizer = self.setup(model, optimizer) + print(type(optimizer.optimizer)) + scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) for epoch in range(1, args.epochs + 1): self.train(args, model, train_loader, optimizer, epoch) From 01720a51e74703c87cb2f177503cdc553c1474d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 15 Oct 2021 13:57:40 +0200 Subject: [PATCH 074/331] remove unused methods --- pytorch_lightning/accelerators/accelerator.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pytorch_lightning/accelerators/accelerator.py b/pytorch_lightning/accelerators/accelerator.py index df611adbf7359..edd80e2747813 100644 --- a/pytorch_lightning/accelerators/accelerator.py +++ b/pytorch_lightning/accelerators/accelerator.py @@ -92,12 +92,6 @@ def setup(self, trainer: "pl.Trainer") -> None: self.setup_optimizers(trainer) self.setup_precision_plugin() - def setup_dataloader(self, dataloader: DataLoader) -> DataLoader: - return self.training_type_plugin.setup_dataloader(dataloader) - - def setup_model(self, model: Module) -> Module: - return self.training_type_plugin.setup_model_and_optimizers(model, None) - def start_training(self, trainer: "pl.Trainer") -> None: """ .. deprecated:: v1.5 From bf48e3d5edb8d57517e8eafb04985657ff4840de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 15 Oct 2021 17:33:58 +0200 Subject: [PATCH 075/331] x x x x x rev --- .../simple/deepspeed_mnist_example.py | 9 +++++---- pytorch_lightning/lite/lite.py | 4 ++-- pytorch_lightning/lite/wrappers.py | 20 ------------------- .../plugins/precision/deepspeed_precision.py | 8 +++++--- .../plugins/training_type/deepspeed.py | 11 ++++------ 5 files changed, 16 insertions(+), 36 deletions(-) diff --git a/pl_examples/lite_examples/simple/deepspeed_mnist_example.py b/pl_examples/lite_examples/simple/deepspeed_mnist_example.py index 15b1fc036e9c6..9f5f5ce89edbd 100644 --- a/pl_examples/lite_examples/simple/deepspeed_mnist_example.py +++ b/pl_examples/lite_examples/simple/deepspeed_mnist_example.py @@ -81,10 +81,11 @@ def train(self, args, model, train_loader, optimizer, epoch): optimizer.zero_grad() output = model(data) loss = F.nll_loss(output, target) - # DEEPSPEED requires you to change loss.backward() to model.backward(loss) - model.backward(loss) - # DEEPSPEED requires you to change optimizer.step() to model.backward(loss) - model.step() + # DEEPSPEED will call model.backward(loss) internally + self.backward(loss) + # DEEPSPEED will call model.step() internally + optimizer.step() + if batch_idx % args.log_interval == 0: print( "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format( diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index ac02fcb9feb99..9bef612ed37e3 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -28,7 +28,7 @@ from pytorch_lightning import Trainer from pytorch_lightning.accelerators import Accelerator, TPUAccelerator from pytorch_lightning.lite.wrappers import _LiteOptimizer, _LiteModule -from pytorch_lightning.plugins import PLUGIN_INPUT, DDPSpawnPlugin, TrainingTypePlugin +from pytorch_lightning.plugins import PLUGIN_INPUT, DDPSpawnPlugin, TrainingTypePlugin, DeepSpeedPlugin from pytorch_lightning.trainer.connectors.accelerator_connector import AcceleratorConnector from pytorch_lightning.trainer.data_loading import TrainerDataLoadingMixin from pytorch_lightning.utilities import move_data_to_device @@ -132,7 +132,7 @@ def setup_dataloader(self, *dataloaders: DataLoader) -> Union[DataLoader, Sequen def backward(self, tensor: Tensor, *args: Any, **kwargs: Any) -> None: # user will call self.backward(loss) instead of loss.backward() - self._accelerator.run_backward(tensor, None, *args, **kwargs) + self._accelerator.run_backward(tensor, self._training_type_plugin.model, *args, **kwargs) @contextmanager def forward_context(self) -> Generator[None, None, None]: diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 3e1dbbd818c1d..1fddafa41f800 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -34,10 +34,6 @@ def optimizer(self) -> Optimizer: return self._optimizer def step(self, closure: Optional[Callable] = None) -> None: - if isinstance(self._accelerator.training_type_plugin, DeepSpeedPlugin): - self._optimizer.step(closure) - return - self._accelerator.optimizer_step( self._optimizer, lambda_closure=closure, @@ -61,19 +57,3 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: output = apply_to_collection(output, function=lambda t: t.to(torch.get_default_dtype()), dtype=Tensor) return output - - def backward(self, loss, *args: Any, **kwargs: Any) -> None: - if not isinstance(self._accelerator.training_type_plugin, DeepSpeedPlugin): - raise RuntimeError( - f"Calling `.backward()` on {self.module.__class__.__name__} is not allowed." - f" Please change your code to call `backward()` on the loss tensor directly." - ) - self._accelerator.run_backward(loss, self.module, *args, **kwargs) - - def step(self) -> None: - if not isinstance(self._accelerator.training_type_plugin, DeepSpeedPlugin): - raise RuntimeError( - f"Calling `.step()` on {self.module.__class__.__name__} is not allowed." - f" Please change your code to call the optimizer's `step()` method instead." - ) - self.module.step() diff --git a/pytorch_lightning/plugins/precision/deepspeed_precision.py b/pytorch_lightning/plugins/precision/deepspeed_precision.py index 774fbe2a74ab1..4f45d8ec9912c 100644 --- a/pytorch_lightning/plugins/precision/deepspeed_precision.py +++ b/pytorch_lightning/plugins/precision/deepspeed_precision.py @@ -52,9 +52,11 @@ def pre_optimizer_step( "Skipping backward by returning `None` from your `training_step` is not supported by `DeepSpeed`" ) # the following should be in a `optimizer_step` hook but we don't have one in the precision plugin. - deepspeed_engine = model.trainer.model if isinstance(model, pl.LightningModule) else model - deepspeed_engine.step() - return False + if isinstance(model, pl.LightningModule): + deepspeed_engine = model.trainer.model + deepspeed_engine.step() + return False + return True def backward(self, model: "pl.LightningModule", closure_loss: Tensor, *args: Any, **kwargs: Any) -> None: if is_overridden("backward", model): diff --git a/pytorch_lightning/plugins/training_type/deepspeed.py b/pytorch_lightning/plugins/training_type/deepspeed.py index 764bcfab18b87..55169e88b361f 100644 --- a/pytorch_lightning/plugins/training_type/deepspeed.py +++ b/pytorch_lightning/plugins/training_type/deepspeed.py @@ -384,23 +384,20 @@ def pre_dispatch(self): def setup_models_and_optimizers( self, models: Sequence[Module], optimizers: Sequence[Optimizer] ) -> Tuple[Sequence[Module], Sequence[Optimizer]]: - if len(models) != len(optimizers): + if not (len(models) == len(optimizers) == 1): raise ValueError( - f"DeepSpeed requires one optimizer per model." + f"Currently only model and one optimizer is supported with DeepSpeed." f" Got {len(models)} models and {len(optimizers)} optimizers instead." ) # TODO: is this the correct place to set this? self.config["train_micro_batch_size_per_gpu"] = 1 - models_and_optimizers = [ - self._setup_model_and_optimizer(model, optimizer) for model, optimizer in zip(models, optimizers) - ] - models, optimizers = zip(*models_and_optimizers) + self._model, optimizer = self._setup_model_and_optimizer(models[0], optimizers[0]) # TODO: do we need to call it here? # self._set_deepspeed_activation_checkpointing() - return list(models), list(optimizers) + return [self._model], [optimizer] def _setup_model_and_optimizer(self, model: Module, optimizer: Optimizer) -> Tuple[DeepSpeedEngine, Optimizer]: # TODO: shouldn't this be optimizer.parameters? From 4572e180efa73245fa2e40c927d592526c3b4d25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 15 Oct 2021 17:40:13 +0200 Subject: [PATCH 076/331] one --- pytorch_lightning/plugins/training_type/deepspeed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/plugins/training_type/deepspeed.py b/pytorch_lightning/plugins/training_type/deepspeed.py index 55169e88b361f..8e1d9fcc1f3e2 100644 --- a/pytorch_lightning/plugins/training_type/deepspeed.py +++ b/pytorch_lightning/plugins/training_type/deepspeed.py @@ -386,7 +386,7 @@ def setup_models_and_optimizers( ) -> Tuple[Sequence[Module], Sequence[Optimizer]]: if not (len(models) == len(optimizers) == 1): raise ValueError( - f"Currently only model and one optimizer is supported with DeepSpeed." + f"Currently only one model and one optimizer is supported with DeepSpeed." f" Got {len(models)} models and {len(optimizers)} optimizers instead." ) From 0c14bb73fdf9b1fc15a0cd3b1ef050fc76e5b934 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 15 Oct 2021 17:41:52 +0200 Subject: [PATCH 077/331] Update pyproject.toml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Carlos Mocholí --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fbc727546682c..22b884b96945c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ module = [ "pytorch_lightning.callbacks.model_summary", "pytorch_lightning.callbacks.pruning", "pytorch_lightning.callbacks.rich_model_summary", - "pytorch_lightning.lite", + "pytorch_lightning.lite.*", "pytorch_lightning.loops.optimization.*", "pytorch_lightning.loops.evaluation_loop", "pytorch_lightning.trainer.connectors.checkpoint_connector", From fc66441fc2a0a43229d00388e8511b71f73bc08e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 16 Oct 2021 02:16:21 +0200 Subject: [PATCH 078/331] remove unsupported args --- pytorch_lightning/lite/lite.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 9bef612ed37e3..9ae9672de441a 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -44,13 +44,11 @@ def __init__( plugins: Optional[Union[PLUGIN_INPUT, List[PLUGIN_INPUT]]] = None, gpus: Optional[Union[List[int], str, int]] = None, tpu_cores: Optional[Union[List[int], str, int]] = None, - ipus: Optional[int] = None, num_processes: int = 1, devices: Optional[Union[List[int], str, int]] = None, num_nodes: int = 1, precision: Union[int, str] = 32, amp_backend: str = "native", - amp_level: Optional[str] = None, replace_sampler_ddp: bool = True, ) -> None: gpu_ids, tpu_cores = Trainer._parse_devices(gpus=gpus, auto_select_gpus=False, tpu_cores=tpu_cores) @@ -58,8 +56,8 @@ def __init__( num_processes=num_processes, devices=devices, tpu_cores=tpu_cores, - ipus=ipus, - distributed_backend=None, # TODO: remove + ipus=None, + distributed_backend=None, accelerator=accelerator, strategy=strategy, gpus=gpus, @@ -71,7 +69,7 @@ def __init__( deterministic=False, precision=precision, amp_type=amp_backend, - amp_level=amp_level, + amp_level=None, plugins=plugins, ) self._accelerator = self._accelerator_connector.select_accelerator() From cb0adae22cdb8c036f818db25abda469a2c9d4bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 16 Oct 2021 02:16:53 +0200 Subject: [PATCH 079/331] setup dataloaders and sampler --- pytorch_lightning/lite/lite.py | 49 +++++++++++++++------------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 9ae9672de441a..ef7088ad84322 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -49,7 +49,6 @@ def __init__( num_nodes: int = 1, precision: Union[int, str] = 32, amp_backend: str = "native", - replace_sampler_ddp: bool = True, ) -> None: gpu_ids, tpu_cores = Trainer._parse_devices(gpus=gpus, auto_select_gpus=False, tpu_cores=tpu_cores) self._accelerator_connector = AcceleratorConnector( @@ -65,7 +64,7 @@ def __init__( num_nodes=num_nodes, sync_batchnorm=False, # TODO: add support? benchmark=False, - replace_sampler_ddp=replace_sampler_ddp, + replace_sampler_ddp=True, deterministic=False, precision=precision, amp_type=amp_backend, @@ -117,17 +116,31 @@ def setup( optimizers = optimizers[0] if len(optimizers) == 1 else optimizers return models, optimizers - def setup_dataloader(self, *dataloaders: DataLoader) -> Union[DataLoader, Sequence[DataLoader]]: + def setup_dataloaders( + self, *dataloaders: DataLoader, replace_sampler: bool = True + ) -> Union[DataLoader, Sequence[DataLoader]]: # user can call this method independently instead of the general purpose setup method # dataloaders = [self._training_type_plugin.setup_dataloader(dataloader) for dataloader in dataloaders] - dataloaders = [ - TrainerDataLoadingMixin._update_dataloader(dataloader, sampler=self._resolve_sampler(dataloader)) - for dataloader in dataloaders - if self._requires_distributed_sampler(dataloader) or isinstance(self._accelerator, TPUAccelerator) - ] + dataloaders = [self.setup_dataloader(dataloader, replace_sampler=replace_sampler) for dataloader in dataloaders] dataloaders = dataloaders[0] if len(dataloaders) == 1 else dataloaders return dataloaders + def setup_dataloader(self, dataloader: DataLoader, replace_sampler: bool = True) -> DataLoader: + if not replace_sampler or not ( + self._requires_distributed_sampler(dataloader) or isinstance(self._accelerator, TPUAccelerator) + ): + return dataloader + if not isinstance(dataloader.sampler, (SequentialSampler, RandomSampler)): + raise MisconfigurationException( + "You seem to have configured a sampler in your DataLoader. This will be replaced " + " by `DistributedSampler` since `replace_sampler_ddp` is True and you are using" + " distributed training. Either remove the sampler from your DataLoader or set" + " `replace_sampler=False` if you want to use your custom sampler." + ) + + sampler = self._get_distributed_sampler(dataloader, **self._training_type_plugin.distributed_sampler_kwargs) + return TrainerDataLoadingMixin._update_dataloader(dataloader, sampler) + def backward(self, tensor: Tensor, *args: Any, **kwargs: Any) -> None: # user will call self.backward(loss) instead of loss.backward() self._accelerator.run_backward(tensor, self._training_type_plugin.model, *args, **kwargs) @@ -178,33 +191,15 @@ def _setup_models_and_optimizers( optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self._accelerator) for optimizer in optimizers] return models, optimizers - # TODO: copied from data_loading.py def _requires_distributed_sampler(self, dataloader: DataLoader) -> bool: return ( - self._accelerator_connector.replace_sampler_ddp - and self._accelerator_connector.is_distributed + self._accelerator_connector.is_distributed and not isinstance(dataloader.sampler, DistributedSampler) and not has_iterable_dataset(dataloader) ) - # TODO: copied and adapted from data_loading.py - def _resolve_sampler(self, dataloader: DataLoader) -> Sampler: - if self._requires_distributed_sampler(dataloader): - if not isinstance(dataloader.sampler, (SequentialSampler, RandomSampler)): - raise MisconfigurationException( - "You seem to have configured a sampler in your DataLoader. This will be replaced " - " by `DistributedSampler` since `replace_sampler_ddp` is True and you are using" - " distributed training. Either remove the sampler from your DataLoader or set" - " `replace_sampler_ddp=False` if you want to use your custom sampler." - ) - return self._get_distributed_sampler(dataloader, **self._training_type_plugin.distributed_sampler_kwargs) - - return dataloader.sampler - - # TODO: copied and adapted from data_loading.py @staticmethod def _get_distributed_sampler(dataloader: DataLoader, **kwargs: Any) -> DistributedSampler: - """This function is used to created the distributed sampler injected within the user DataLoader.""" kwargs.setdefault("seed", int(os.getenv("PL_GLOBAL_SEED", 0))) sampler = DistributedSampler(dataloader.dataset, **kwargs) return sampler From 61839eade33680028bf6e6a74360f69dd9c9be2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 16 Oct 2021 02:20:26 +0200 Subject: [PATCH 080/331] update example --- pl_examples/lite_examples/simple/deepspeed_mnist_example.py | 2 +- pl_examples/lite_examples/simple/mnist_example.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pl_examples/lite_examples/simple/deepspeed_mnist_example.py b/pl_examples/lite_examples/simple/deepspeed_mnist_example.py index 9f5f5ce89edbd..ce20e8d4bd22a 100644 --- a/pl_examples/lite_examples/simple/deepspeed_mnist_example.py +++ b/pl_examples/lite_examples/simple/deepspeed_mnist_example.py @@ -61,7 +61,7 @@ def run(self, args): model = Net().to(self.device) optimizer = optim.Adadelta(model.parameters(), lr=args.lr) - train_loader, test_loader = self.setup_dataloader(train_loader, test_loader) + train_loader, test_loader = self.setup_dataloaders(train_loader, test_loader) model, optimizer = self.setup(model, optimizer) scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) diff --git a/pl_examples/lite_examples/simple/mnist_example.py b/pl_examples/lite_examples/simple/mnist_example.py index 5849b2d86ee77..352fb6118b95f 100644 --- a/pl_examples/lite_examples/simple/mnist_example.py +++ b/pl_examples/lite_examples/simple/mnist_example.py @@ -62,7 +62,7 @@ def run(self, args): model = Net().to(self.device) optimizer = optim.Adadelta(model.parameters(), lr=args.lr) - train_loader, test_loader = self.setup_dataloader(train_loader, test_loader) + train_loader, test_loader = self.setup_dataloaders(train_loader, test_loader) assert isinstance(train_loader.sampler, DistributedSampler) assert isinstance(test_loader.sampler, DistributedSampler) model, optimizer = self.setup(model, optimizer) From c8d8cc9a642d568ac0d7b31f04a10fdc1fe2f3a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 16 Oct 2021 02:45:17 +0200 Subject: [PATCH 081/331] update type hint --- pytorch_lightning/plugins/training_type/sharded.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/plugins/training_type/sharded.py b/pytorch_lightning/plugins/training_type/sharded.py index 34021ba15a2b7..fd24cfef2a327 100644 --- a/pytorch_lightning/plugins/training_type/sharded.py +++ b/pytorch_lightning/plugins/training_type/sharded.py @@ -63,7 +63,9 @@ def configure_ddp(self) -> None: trainer.optimizers = optimizers trainer.convert_to_lightning_optimizers() - def _reinit_optimizers_with_oss(self, optimizers: Sequence[Union[Optimizer, LightningOptimizer]]) -> Sequence[OSS]: + def _reinit_optimizers_with_oss( + self, optimizers: Sequence[Union[Optimizer, LightningOptimizer]] + ) -> Sequence["OSS"]: for x, optimizer in enumerate(optimizers): if isinstance(optimizer, LightningOptimizer): optimizer = optimizer._optimizer @@ -83,7 +85,7 @@ def _reinit_optimizers_with_oss(self, optimizers: Sequence[Union[Optimizer, Ligh del optimizer return optimizers - def _wrap_optimizers(self, optimizers: Sequence[Optimizer]) -> Sequence[OSS]: + def _wrap_optimizers(self, optimizers: Sequence[Optimizer]) -> Sequence["OSS"]: if self.model is not None and self.model.trainer.state.fn != TrainerFn.FITTING: return optimizers From 929aa6a3ceb6fb0ac1be90ba40d0b9b8207dcc10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 16 Oct 2021 03:02:33 +0200 Subject: [PATCH 082/331] limit input choices --- .../gan/deepspeed_gan_example.py | 2 +- pl_examples/lite_examples/gan/gan_example.py | 2 +- pl_examples/lite_examples/gan/run_examples.py | 2 +- .../simple/deepspeed_mnist_example.py | 2 +- .../lite_examples/simple/mnist_example.py | 9 ++++- pytorch_lightning/lite/lite.py | 40 ++++++++++++++++--- 6 files changed, 46 insertions(+), 11 deletions(-) diff --git a/pl_examples/lite_examples/gan/deepspeed_gan_example.py b/pl_examples/lite_examples/gan/deepspeed_gan_example.py index 2cd6739562cb9..8e9cc5ef4bd8e 100644 --- a/pl_examples/lite_examples/gan/deepspeed_gan_example.py +++ b/pl_examples/lite_examples/gan/deepspeed_gan_example.py @@ -172,5 +172,5 @@ def run(self): if __name__ == "__main__": - gan = GANTrainer(gpus=1, accelerator="deepspeed") + gan = GANTrainer(gpus=1, strategy="deepspeed", accelerator="gpu") gan.run() diff --git a/pl_examples/lite_examples/gan/gan_example.py b/pl_examples/lite_examples/gan/gan_example.py index c4c09feb52522..6cfd75ad42f74 100644 --- a/pl_examples/lite_examples/gan/gan_example.py +++ b/pl_examples/lite_examples/gan/gan_example.py @@ -173,5 +173,5 @@ def run(self): if __name__ == "__main__": - gan = GANTrainer(accelerator="ddp", num_processes=2) + gan = GANTrainer(accelerator="ddp", devices=2) gan.run() diff --git a/pl_examples/lite_examples/gan/run_examples.py b/pl_examples/lite_examples/gan/run_examples.py index 15bebf7933319..8d77e7f14b7d2 100644 --- a/pl_examples/lite_examples/gan/run_examples.py +++ b/pl_examples/lite_examples/gan/run_examples.py @@ -6,7 +6,7 @@ parser = argparse.ArgumentParser() parser.add_argument("--accelerator", type=str, default=None) parser.add_argument("--gpus", type=int, default=None) - parser.add_argument("--num_processes", type=int, default=1) + parser.add_argument("--devices", type=int, default=1) parser.add_argument("--precision", type=int, default=32) args = parser.parse_args() diff --git a/pl_examples/lite_examples/simple/deepspeed_mnist_example.py b/pl_examples/lite_examples/simple/deepspeed_mnist_example.py index ce20e8d4bd22a..6235e13c8121c 100644 --- a/pl_examples/lite_examples/simple/deepspeed_mnist_example.py +++ b/pl_examples/lite_examples/simple/deepspeed_mnist_example.py @@ -148,7 +148,7 @@ def main(): parser.add_argument("--precision", type=int, default=32) args = parser.parse_args() - mnist = DeepSpeedMNIST(gpus=2, strategy="deepspeed", num_processes=1, precision=args.precision) + mnist = DeepSpeedMNIST(devices=2, strategy="deepspeed", precision=args.precision) mnist.run(args) diff --git a/pl_examples/lite_examples/simple/mnist_example.py b/pl_examples/lite_examples/simple/mnist_example.py index 352fb6118b95f..eb3ff1780d777 100644 --- a/pl_examples/lite_examples/simple/mnist_example.py +++ b/pl_examples/lite_examples/simple/mnist_example.py @@ -144,13 +144,18 @@ def main(): ) parser.add_argument("--save-model", action="store_true", default=False, help="For Saving the current Model") parser.add_argument("--accelerator", type=str, default=None) + parser.add_argument("--strategy", type=str, default=None) parser.add_argument("--gpus", type=int, default=None) - parser.add_argument("--num_processes", type=int, default=1) + parser.add_argument("--devices", type=int, default=1) parser.add_argument("--precision", type=int, default=32) args = parser.parse_args() mnist = MNIST( - gpus=args.gpus, accelerator=args.accelerator, num_processes=args.num_processes, precision=args.precision + gpus=args.gpus, + devices=args.devices, + accelerator=args.accelerator, + strategy=args.strategy, + precision=args.precision, ) mnist.run(args) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index ef7088ad84322..22a54ae054194 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -31,7 +31,7 @@ from pytorch_lightning.plugins import PLUGIN_INPUT, DDPSpawnPlugin, TrainingTypePlugin, DeepSpeedPlugin from pytorch_lightning.trainer.connectors.accelerator_connector import AcceleratorConnector from pytorch_lightning.trainer.data_loading import TrainerDataLoadingMixin -from pytorch_lightning.utilities import move_data_to_device +from pytorch_lightning.utilities import move_data_to_device, DistributedType, DeviceType from pytorch_lightning.utilities.data import has_iterable_dataset from pytorch_lightning.utilities.exceptions import MisconfigurationException @@ -44,15 +44,23 @@ def __init__( plugins: Optional[Union[PLUGIN_INPUT, List[PLUGIN_INPUT]]] = None, gpus: Optional[Union[List[int], str, int]] = None, tpu_cores: Optional[Union[List[int], str, int]] = None, - num_processes: int = 1, devices: Optional[Union[List[int], str, int]] = None, num_nodes: int = 1, precision: Union[int, str] = 32, - amp_backend: str = "native", ) -> None: + if not isinstance(accelerator, Accelerator) or accelerator not in self._supported_device_types(): + raise MisconfigurationException( + f"`accelerator={repr(accelerator)}` is not a valid choice." + f" Choose one of {self._supported_device_types()} or pass in a `Accelerator` instance." + ) + if not isinstance(strategy, TrainingTypePlugin) or strategy not in self._supported_strategy_types(): + raise MisconfigurationException( + f"`strategy={repr(strategy)}` is not a valid choice." + f" Choose one of {self._supported_strategy_types()} or pass in a `TrainingTypePlugin` instance." + ) gpu_ids, tpu_cores = Trainer._parse_devices(gpus=gpus, auto_select_gpus=False, tpu_cores=tpu_cores) self._accelerator_connector = AcceleratorConnector( - num_processes=num_processes, + num_processes=1, devices=devices, tpu_cores=tpu_cores, ipus=None, @@ -67,7 +75,7 @@ def __init__( replace_sampler_ddp=True, deterministic=False, precision=precision, - amp_type=amp_backend, + amp_type="native", amp_level=None, plugins=plugins, ) @@ -203,3 +211,25 @@ def _get_distributed_sampler(dataloader: DataLoader, **kwargs: Any) -> Distribut kwargs.setdefault("seed", int(os.getenv("PL_GLOBAL_SEED", 0))) sampler = DistributedSampler(dataloader.dataset, **kwargs) return sampler + + @staticmethod + def _supported_device_types() -> Sequence[DeviceType]: + return ( + None, + DeviceType.CPU, + DeviceType.GPU, + DeviceType.TPU, + ) + + @staticmethod + def _supported_strategy_types() -> Sequence[DistributedType]: + return ( + None, + DistributedType.DP, + DistributedType.DDP, + DistributedType.DDP_SPAWN, + DistributedType.TPU_SPAWN, + DistributedType.DP, + DistributedType.DEEPSPEED, + DistributedType.DDP_SHARDED, + ) From 44a158543d3aea424ebab55cbd83bb9d1e3fe059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 16 Oct 2021 03:04:31 +0200 Subject: [PATCH 083/331] update error --- pytorch_lightning/lite/lite.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 22a54ae054194..cb809bad6e848 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -216,20 +216,20 @@ def _get_distributed_sampler(dataloader: DataLoader, **kwargs: Any) -> Distribut def _supported_device_types() -> Sequence[DeviceType]: return ( None, - DeviceType.CPU, - DeviceType.GPU, - DeviceType.TPU, + DeviceType.CPU.value, + DeviceType.GPU.value, + DeviceType.TPU.value, ) @staticmethod - def _supported_strategy_types() -> Sequence[DistributedType]: + def _supported_strategy_types() -> Sequence[str]: return ( None, - DistributedType.DP, - DistributedType.DDP, - DistributedType.DDP_SPAWN, - DistributedType.TPU_SPAWN, - DistributedType.DP, - DistributedType.DEEPSPEED, - DistributedType.DDP_SHARDED, + DistributedType.DP.value, + DistributedType.DDP.value, + DistributedType.DDP_SPAWN.value, + DistributedType.TPU_SPAWN.value, + DistributedType.DP.value, + DistributedType.DEEPSPEED.value, + DistributedType.DDP_SHARDED.value, ) From 46eb567a7a716722d67dbe9bf767d8218e53831b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 16 Oct 2021 03:30:00 +0200 Subject: [PATCH 084/331] update input validation --- .../gan/deepspeed_gan_example.py | 2 +- pl_examples/lite_examples/gan/gan_example.py | 2 +- pytorch_lightning/lite/lite.py | 80 +++++++++++-------- 3 files changed, 47 insertions(+), 37 deletions(-) diff --git a/pl_examples/lite_examples/gan/deepspeed_gan_example.py b/pl_examples/lite_examples/gan/deepspeed_gan_example.py index 8e9cc5ef4bd8e..2e4281c969ec9 100644 --- a/pl_examples/lite_examples/gan/deepspeed_gan_example.py +++ b/pl_examples/lite_examples/gan/deepspeed_gan_example.py @@ -53,7 +53,7 @@ class GANTrainer(LightningLite): def run(self): - print("selected plugin: ", self._training_type_plugin) + print("selected plugin: ", self._strategy) seed_everything(123) # TODO: how do we handle this in Accelerator? diff --git a/pl_examples/lite_examples/gan/gan_example.py b/pl_examples/lite_examples/gan/gan_example.py index 6cfd75ad42f74..dc197004df3ba 100644 --- a/pl_examples/lite_examples/gan/gan_example.py +++ b/pl_examples/lite_examples/gan/gan_example.py @@ -54,7 +54,7 @@ class GANTrainer(LightningLite): def run(self): - print("selected plugin: ", self._training_type_plugin) + print("selected plugin: ", self._strategy) seed_everything(123) # TODO: how do we handle this in Accelerator? diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index cb809bad6e848..12fe1b5f4cae5 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -48,16 +48,8 @@ def __init__( num_nodes: int = 1, precision: Union[int, str] = 32, ) -> None: - if not isinstance(accelerator, Accelerator) or accelerator not in self._supported_device_types(): - raise MisconfigurationException( - f"`accelerator={repr(accelerator)}` is not a valid choice." - f" Choose one of {self._supported_device_types()} or pass in a `Accelerator` instance." - ) - if not isinstance(strategy, TrainingTypePlugin) or strategy not in self._supported_strategy_types(): - raise MisconfigurationException( - f"`strategy={repr(strategy)}` is not a valid choice." - f" Choose one of {self._supported_strategy_types()} or pass in a `TrainingTypePlugin` instance." - ) + self._check_accelerator_support(accelerator) + self._check_strategy_support(strategy) gpu_ids, tpu_cores = Trainer._parse_devices(gpus=gpus, auto_select_gpus=False, tpu_cores=tpu_cores) self._accelerator_connector = AcceleratorConnector( num_processes=1, @@ -80,7 +72,7 @@ def __init__( plugins=plugins, ) self._accelerator = self._accelerator_connector.select_accelerator() - self._training_type_plugin = self._accelerator.training_type_plugin + self._strategy = self._accelerator.training_type_plugin self._precision_plugin = self._accelerator.precision_plugin # wrap the run method so we can inject setup logic or spawn processes for the user @@ -92,19 +84,19 @@ def device(self) -> torch.device: @property def global_rank(self) -> int: - return getattr(self._training_type_plugin, "global_rank", 0) + return getattr(self._strategy, "global_rank", 0) @property def local_rank(self) -> int: - return getattr(self._training_type_plugin, "local_rank", 0) + return getattr(self._strategy, "local_rank", 0) @property def node_rank(self) -> int: - return getattr(self._training_type_plugin, "node_rank", 0) + return getattr(self._strategy, "node_rank", 0) @property def world_size(self) -> int: - return getattr(self._training_type_plugin, "world_size", 1) + return getattr(self._strategy, "world_size", 1) @abstractmethod def run(self, *args: Any, **kwargs: Any) -> None: @@ -128,7 +120,7 @@ def setup_dataloaders( self, *dataloaders: DataLoader, replace_sampler: bool = True ) -> Union[DataLoader, Sequence[DataLoader]]: # user can call this method independently instead of the general purpose setup method - # dataloaders = [self._training_type_plugin.setup_dataloader(dataloader) for dataloader in dataloaders] + # dataloaders = [self._strategy.setup_dataloader(dataloader) for dataloader in dataloaders] dataloaders = [self.setup_dataloader(dataloader, replace_sampler=replace_sampler) for dataloader in dataloaders] dataloaders = dataloaders[0] if len(dataloaders) == 1 else dataloaders return dataloaders @@ -146,12 +138,12 @@ def setup_dataloader(self, dataloader: DataLoader, replace_sampler: bool = True) " `replace_sampler=False` if you want to use your custom sampler." ) - sampler = self._get_distributed_sampler(dataloader, **self._training_type_plugin.distributed_sampler_kwargs) + sampler = self._get_distributed_sampler(dataloader, **self._strategy.distributed_sampler_kwargs) return TrainerDataLoadingMixin._update_dataloader(dataloader, sampler) def backward(self, tensor: Tensor, *args: Any, **kwargs: Any) -> None: # user will call self.backward(loss) instead of loss.backward() - self._accelerator.run_backward(tensor, self._training_type_plugin.model, *args, **kwargs) + self._accelerator.run_backward(tensor, self._strategy.model, *args, **kwargs) @contextmanager def forward_context(self) -> Generator[None, None, None]: @@ -168,7 +160,7 @@ def print(self, *args: Any, **kwargs: Any) -> None: print(*args, **kwargs) def reduce_decision(self, decision: bool) -> bool: - return self._training_type_plugin.reduce_boolean_decision(decision) + return self._strategy.reduce_boolean_decision(decision) def save_checkpoint(self, filepath: Union[str, Path], content: Dict[str, Any]) -> None: raise NotImplementedError() @@ -181,9 +173,9 @@ def _run_wrapper(self, run_method: Callable) -> Callable: return partial(self._run_impl, run_method) def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> None: - self._training_type_plugin.setup_environment() - if isinstance(self._training_type_plugin, DDPSpawnPlugin): - self._training_type_plugin.spawn(run_method, *args, **kwargs) + self._strategy.setup_environment() + if isinstance(self._strategy, DDPSpawnPlugin): + self._strategy.spawn(run_method, *args, **kwargs) else: run_method(*args, **kwargs) # TODO: any teardown needed here? @@ -194,7 +186,7 @@ def _setup_models_and_optimizers( optimizers: Sequence[Optimizer], ) -> Tuple[Sequence[_LiteModule], Sequence[_LiteOptimizer]]: # Let accelerator/plugin wrap and connect the models and optimizers - models, optimizers = self._training_type_plugin.setup_models_and_optimizers(models, optimizers) + models, optimizers = self._strategy.setup_models_and_optimizers(models, optimizers) models = [_LiteModule(module=model, accelerator=self._accelerator) for model in models] optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self._accelerator) for optimizer in optimizers] return models, optimizers @@ -212,24 +204,42 @@ def _get_distributed_sampler(dataloader: DataLoader, **kwargs: Any) -> Distribut sampler = DistributedSampler(dataloader.dataset, **kwargs) return sampler + def _check_accelerator_support(self, accelerator: Optional[Union[str, Accelerator]]) -> None: + if accelerator is None: + return + supported = [t.lower() for t in self._supported_device_types()] + if not isinstance(accelerator, (Accelerator, str)) or accelerator not in supported: + raise MisconfigurationException( + f"`accelerator={repr(accelerator)}` is not a valid choice." + f" Choose one of {supported} or pass in a `Accelerator` instance." + ) + + def _check_strategy_support(self, strategy: Optional[Union[str, TrainingTypePlugin]]) -> None: + if strategy is None: + return + supported = [t.lower() for t in self._supported_strategy_types()] + if not isinstance(strategy, (TrainingTypePlugin, str)) or strategy not in supported: + raise MisconfigurationException( + f"`strategy={repr(strategy)}` is not a valid choice." + f" Choose one of {supported} or pass in a `TrainingTypePlugin` instance." + ) + @staticmethod def _supported_device_types() -> Sequence[DeviceType]: return ( - None, - DeviceType.CPU.value, - DeviceType.GPU.value, - DeviceType.TPU.value, + DeviceType.CPU, + DeviceType.GPU, + DeviceType.TPU, ) @staticmethod def _supported_strategy_types() -> Sequence[str]: return ( - None, - DistributedType.DP.value, - DistributedType.DDP.value, - DistributedType.DDP_SPAWN.value, - DistributedType.TPU_SPAWN.value, - DistributedType.DP.value, - DistributedType.DEEPSPEED.value, - DistributedType.DDP_SHARDED.value, + DistributedType.DP, + DistributedType.DDP, + DistributedType.DDP_SPAWN, + DistributedType.TPU_SPAWN, + DistributedType.DP, + DistributedType.DEEPSPEED, + DistributedType.DDP_SHARDED, ) From 59a44cbe99568e9de12c98f398bfd3ee028c9924 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 16 Oct 2021 12:28:15 +0200 Subject: [PATCH 085/331] move model to device automatically --- pl_examples/lite_examples/gan/gan_example.py | 4 ++-- pl_examples/lite_examples/simple/deepspeed_mnist_example.py | 2 +- pl_examples/lite_examples/simple/mnist_example.py | 2 +- pytorch_lightning/lite/lite.py | 4 ++++ 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pl_examples/lite_examples/gan/gan_example.py b/pl_examples/lite_examples/gan/gan_example.py index dc197004df3ba..879932971a96f 100644 --- a/pl_examples/lite_examples/gan/gan_example.py +++ b/pl_examples/lite_examples/gan/gan_example.py @@ -89,8 +89,8 @@ def run(self): netD = Discriminator() netD.apply(weights_init) - self.to_device(netG) - self.to_device(netD) + # self.to_device(netG) + # self.to_device(netD) criterion = nn.BCELoss() diff --git a/pl_examples/lite_examples/simple/deepspeed_mnist_example.py b/pl_examples/lite_examples/simple/deepspeed_mnist_example.py index 6235e13c8121c..793e729f371b6 100644 --- a/pl_examples/lite_examples/simple/deepspeed_mnist_example.py +++ b/pl_examples/lite_examples/simple/deepspeed_mnist_example.py @@ -58,7 +58,7 @@ def run(self, args): train_loader = torch.utils.data.DataLoader(dataset1, **train_kwargs) test_loader = torch.utils.data.DataLoader(dataset2, **test_kwargs) - model = Net().to(self.device) + model = Net() optimizer = optim.Adadelta(model.parameters(), lr=args.lr) train_loader, test_loader = self.setup_dataloaders(train_loader, test_loader) diff --git a/pl_examples/lite_examples/simple/mnist_example.py b/pl_examples/lite_examples/simple/mnist_example.py index eb3ff1780d777..cc9a6ca8edfb6 100644 --- a/pl_examples/lite_examples/simple/mnist_example.py +++ b/pl_examples/lite_examples/simple/mnist_example.py @@ -59,7 +59,7 @@ def run(self, args): train_loader = torch.utils.data.DataLoader(dataset1, **train_kwargs) test_loader = torch.utils.data.DataLoader(dataset2, **test_kwargs) - model = Net().to(self.device) + model = Net() optimizer = optim.Adadelta(model.parameters(), lr=args.lr) train_loader, test_loader = self.setup_dataloaders(train_loader, test_loader) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 12fe1b5f4cae5..df55da7e83679 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -106,12 +106,16 @@ def setup( self, models: Union[nn.Module, Sequence[nn.Module]], optimizers: Union[Optimizer, Sequence[Optimizer]], + move_to_device: bool = True, ) -> Tuple[Union[nn.Module, Sequence[nn.Module]], Union[Optimizer, Sequence[Optimizer]]]: # wrap all objects passed in and return them in the same order models = [models] if isinstance(models, nn.Module) else models optimizers = [optimizers] if isinstance(optimizers, Optimizer) else optimizers models, optimizers = self._setup_models_and_optimizers(models, optimizers) + if move_to_device: + models = [self.to_device(model) for model in models] + models = models[0] if len(models) == 1 else models optimizers = optimizers[0] if len(optimizers) == 1 else optimizers return models, optimizers From fe4746c0616d85be996a30a9f937763240fd5b43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 16 Oct 2021 13:51:04 +0200 Subject: [PATCH 086/331] to device x update --- pl_examples/lite_examples/gan/gan_example.py | 2 +- .../simple/deepspeed_mnist_example.py | 156 ------------------ .../lite_examples/simple/mnist_example.py | 4 +- pytorch_lightning/lite/lite.py | 18 +- pytorch_lightning/lite/wrappers.py | 43 ++++- 5 files changed, 54 insertions(+), 169 deletions(-) delete mode 100644 pl_examples/lite_examples/simple/deepspeed_mnist_example.py diff --git a/pl_examples/lite_examples/gan/gan_example.py b/pl_examples/lite_examples/gan/gan_example.py index 879932971a96f..976928f3e49ca 100644 --- a/pl_examples/lite_examples/gan/gan_example.py +++ b/pl_examples/lite_examples/gan/gan_example.py @@ -114,7 +114,7 @@ def run(self): ########################### # train with real netD.zero_grad() - real_cpu = self.to_device(data[0]) + real_cpu = data[0] batch_size = real_cpu.size(0) label = torch.full((batch_size,), real_label, dtype=real_cpu.dtype, device=self.device) diff --git a/pl_examples/lite_examples/simple/deepspeed_mnist_example.py b/pl_examples/lite_examples/simple/deepspeed_mnist_example.py deleted file mode 100644 index 793e729f371b6..0000000000000 --- a/pl_examples/lite_examples/simple/deepspeed_mnist_example.py +++ /dev/null @@ -1,156 +0,0 @@ -import argparse -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.optim as optim -from torchvision import datasets, transforms -from torch.optim.lr_scheduler import StepLR - -from pytorch_lightning import seed_everything -from pytorch_lightning.lite import LightningLite - - -class Net(nn.Module): - def __init__(self): - super(Net, self).__init__() - self.conv1 = nn.Conv2d(1, 32, 3, 1) - self.conv2 = nn.Conv2d(32, 64, 3, 1) - self.dropout1 = nn.Dropout(0.25) - self.dropout2 = nn.Dropout(0.5) - self.fc1 = nn.Linear(9216, 128) - self.fc2 = nn.Linear(128, 10) - - def forward(self, x): - x = self.conv1(x) - x = F.relu(x) - x = self.conv2(x) - x = F.relu(x) - x = F.max_pool2d(x, 2) - x = self.dropout1(x) - x = torch.flatten(x, 1) - x = self.fc1(x) - x = F.relu(x) - x = self.dropout2(x) - x = self.fc2(x) - output = F.log_softmax(x, dim=1) - return output - - -class DeepSpeedMNIST(LightningLite): - def run(self, args): - use_cuda = self.device.type == "cuda" - - seed_everything(args.seed) - - train_kwargs = {"batch_size": args.batch_size} - test_kwargs = {"batch_size": args.test_batch_size} - if use_cuda: - cuda_kwargs = {"num_workers": 1, "pin_memory": True, "shuffle": True} - train_kwargs.update(cuda_kwargs) - test_kwargs.update(cuda_kwargs) - - if self.local_rank == 0: - datasets.MNIST("../data", download=True) - - transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]) - dataset1 = datasets.MNIST("../data", train=True, transform=transform) - dataset2 = datasets.MNIST("../data", train=False, transform=transform) - train_loader = torch.utils.data.DataLoader(dataset1, **train_kwargs) - test_loader = torch.utils.data.DataLoader(dataset2, **test_kwargs) - - model = Net() - optimizer = optim.Adadelta(model.parameters(), lr=args.lr) - - train_loader, test_loader = self.setup_dataloaders(train_loader, test_loader) - model, optimizer = self.setup(model, optimizer) - - scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) - for epoch in range(1, args.epochs + 1): - self.train(args, model, train_loader, optimizer, epoch) - self.test(model, test_loader) - scheduler.step() - - if args.save_model: - torch.save(model.state_dict(), "mnist_cnn.pt") - - def train(self, args, model, train_loader, optimizer, epoch): - model.train() - for batch_idx, (data, target) in enumerate(train_loader): - data, target = data.to(self.device), target.to(self.device) - # TODO: model.zero_grad() vs. optimizer.zero_grad() with deepspeed? - optimizer.zero_grad() - output = model(data) - loss = F.nll_loss(output, target) - # DEEPSPEED will call model.backward(loss) internally - self.backward(loss) - # DEEPSPEED will call model.step() internally - optimizer.step() - - if batch_idx % args.log_interval == 0: - print( - "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format( - epoch, - batch_idx * len(data), - len(train_loader.dataset), - 100.0 * batch_idx / len(train_loader), - loss.item(), - ) - ) - if args.dry_run: - break - - def test(self, model, test_loader): - model.eval() - test_loss = 0 - correct = 0 - with torch.no_grad(): - for data, target in test_loader: - data, target = data.to(self.device), target.to(self.device) - output = model(data) - test_loss += F.nll_loss(output, target, reduction="sum").item() # sum up batch loss - pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability - correct += pred.eq(target.view_as(pred)).sum().item() - - test_loss /= len(test_loader.dataset) - - print( - "\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n".format( - test_loss, correct, len(test_loader.dataset), 100.0 * correct / len(test_loader.dataset) - ) - ) - - -def main(): - # Training settings - parser = argparse.ArgumentParser(description="PyTorch MNIST Example") - parser.add_argument( - "--batch-size", type=int, default=64, metavar="N", help="input batch size for training (default: 64)" - ) - parser.add_argument( - "--test-batch-size", type=int, default=1000, metavar="N", help="input batch size for testing (default: 1000)" - ) - parser.add_argument("--epochs", type=int, default=14, metavar="N", help="number of epochs to train (default: 14)") - parser.add_argument("--lr", type=float, default=1.0, metavar="LR", help="learning rate (default: 1.0)") - parser.add_argument("--gamma", type=float, default=0.7, metavar="M", help="Learning rate step gamma (default: 0.7)") - parser.add_argument("--dry-run", action="store_true", default=False, help="quickly check a single pass") - parser.add_argument("--seed", type=int, default=1, metavar="S", help="random seed (default: 1)") - parser.add_argument( - "--log-interval", - type=int, - default=10, - metavar="N", - help="how many batches to wait before logging training status", - ) - parser.add_argument("--save-model", action="store_true", default=False, help="For Saving the current Model") - parser.add_argument("--accelerator", type=str, default=None) - parser.add_argument("--gpus", type=int, default=None) - parser.add_argument("--num_processes", type=int, default=1) - parser.add_argument("--precision", type=int, default=32) - args = parser.parse_args() - - mnist = DeepSpeedMNIST(devices=2, strategy="deepspeed", precision=args.precision) - mnist.run(args) - - -if __name__ == "__main__": - main() diff --git a/pl_examples/lite_examples/simple/mnist_example.py b/pl_examples/lite_examples/simple/mnist_example.py index cc9a6ca8edfb6..c0a1931280891 100644 --- a/pl_examples/lite_examples/simple/mnist_example.py +++ b/pl_examples/lite_examples/simple/mnist_example.py @@ -67,8 +67,6 @@ def run(self, args): assert isinstance(test_loader.sampler, DistributedSampler) model, optimizer = self.setup(model, optimizer) - print(type(optimizer.optimizer)) - scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) for epoch in range(1, args.epochs + 1): self.train(args, model, train_loader, optimizer, epoch) @@ -81,7 +79,7 @@ def run(self, args): def train(self, args, model, train_loader, optimizer, epoch): model.train() for batch_idx, (data, target) in enumerate(train_loader): - data, target = data.to(self.device), target.to(self.device) + # data, target = data.to(self.device), target.to(self.device) optimizer.zero_grad() output = model(data) loss = F.nll_loss(output, target) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index df55da7e83679..b0b27a097426f 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -27,7 +27,7 @@ from pytorch_lightning import Trainer from pytorch_lightning.accelerators import Accelerator, TPUAccelerator -from pytorch_lightning.lite.wrappers import _LiteOptimizer, _LiteModule +from pytorch_lightning.lite.wrappers import _LiteOptimizer, _LiteModule, _LiteDataLoader from pytorch_lightning.plugins import PLUGIN_INPUT, DDPSpawnPlugin, TrainingTypePlugin, DeepSpeedPlugin from pytorch_lightning.trainer.connectors.accelerator_connector import AcceleratorConnector from pytorch_lightning.trainer.data_loading import TrainerDataLoadingMixin @@ -121,15 +121,19 @@ def setup( return models, optimizers def setup_dataloaders( - self, *dataloaders: DataLoader, replace_sampler: bool = True + self, *dataloaders: DataLoader, replace_sampler: bool = True, move_to_device: bool = True ) -> Union[DataLoader, Sequence[DataLoader]]: # user can call this method independently instead of the general purpose setup method - # dataloaders = [self._strategy.setup_dataloader(dataloader) for dataloader in dataloaders] - dataloaders = [self.setup_dataloader(dataloader, replace_sampler=replace_sampler) for dataloader in dataloaders] + dataloaders = [ + self.setup_dataloader(dataloader, replace_sampler=replace_sampler, move_to_device=move_to_device) + for dataloader in dataloaders + ] dataloaders = dataloaders[0] if len(dataloaders) == 1 else dataloaders return dataloaders - def setup_dataloader(self, dataloader: DataLoader, replace_sampler: bool = True) -> DataLoader: + def setup_dataloader( + self, dataloader: DataLoader, replace_sampler: bool = True, move_to_device: bool = True + ) -> DataLoader: if not replace_sampler or not ( self._requires_distributed_sampler(dataloader) or isinstance(self._accelerator, TPUAccelerator) ): @@ -143,7 +147,9 @@ def setup_dataloader(self, dataloader: DataLoader, replace_sampler: bool = True) ) sampler = self._get_distributed_sampler(dataloader, **self._strategy.distributed_sampler_kwargs) - return TrainerDataLoadingMixin._update_dataloader(dataloader, sampler) + kwargs = TrainerDataLoadingMixin._get_dataloader_init_kwargs(dataloader, sampler) + device = self.device if move_to_device else None + return _LiteDataLoader(device=device, **kwargs) def backward(self, tensor: Tensor, *args: Any, **kwargs: Any) -> None: # user will call self.backward(loss) instead of loss.backward() diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 1fddafa41f800..b48c5b949f874 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -11,15 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Callable, Optional +from typing import Any, Callable, Optional, Dict import torch from torch import nn as nn, Tensor from torch.optim import Optimizer +from torch.utils.data import DataLoader from pytorch_lightning.accelerators import Accelerator -from pytorch_lightning.plugins import DeepSpeedPlugin -from pytorch_lightning.utilities.apply_func import apply_to_collection +from pytorch_lightning.utilities.apply_func import apply_to_collection, move_data_to_device # TODO: add attributes and methods from Optimizer @@ -57,3 +57,40 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: output = apply_to_collection(output, function=lambda t: t.to(torch.get_default_dtype()), dtype=Tensor) return output + + +class _LiteDataLoader(DataLoader): + def __init__(self, device: Optional[torch.device] = None, **dl_kwargs): + super().__init__(**dl_kwargs) + self._device = device + + def __iter__(self): + iterator = super().__iter__() + if self._device is None: + return iterator + + for item in iterator: + yield move_data_to_device(item, self._device) + + +# +# def iterator_wrapper(iter_method: Callable): +# iterator = iter_method() +# for item in iterator: +# print("additional") +# yield item +# +# +# def iterator_decorator(fn): +# def _it(): +# return iterator_wrapper(fn) +# +# return _it +# +# +# if __name__ == "__main__": +# dset = BoringModel().train_dataloader().dataset +# loader = DataLoader(dset, num_workers=2) +# loader.__iter__ = iterator_decorator(loader.__iter__) +# for x in iter(loader): +# print() From 9ee087fe56190e29276dfd2d46672090d6cc3529 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 16 Oct 2021 14:01:12 +0200 Subject: [PATCH 087/331] typing --- pytorch_lightning/lite/lite.py | 3 +-- pytorch_lightning/lite/wrappers.py | 31 +++++------------------------- 2 files changed, 6 insertions(+), 28 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index b0b27a097426f..3b1d9ccc01936 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -211,8 +211,7 @@ def _requires_distributed_sampler(self, dataloader: DataLoader) -> bool: @staticmethod def _get_distributed_sampler(dataloader: DataLoader, **kwargs: Any) -> DistributedSampler: kwargs.setdefault("seed", int(os.getenv("PL_GLOBAL_SEED", 0))) - sampler = DistributedSampler(dataloader.dataset, **kwargs) - return sampler + return DistributedSampler(dataloader.dataset, **kwargs) def _check_accelerator_support(self, accelerator: Optional[Union[str, Accelerator]]) -> None: if accelerator is None: diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index b48c5b949f874..6f719d4e11a06 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -11,12 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Callable, Optional, Dict +from typing import Any, Callable, Optional, Dict, Generator, Iterator, Union import torch from torch import nn as nn, Tensor from torch.optim import Optimizer from torch.utils.data import DataLoader +from torch.utils.data.dataloader import _BaseDataLoaderIter from pytorch_lightning.accelerators import Accelerator from pytorch_lightning.utilities.apply_func import apply_to_collection, move_data_to_device @@ -60,37 +61,15 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: class _LiteDataLoader(DataLoader): - def __init__(self, device: Optional[torch.device] = None, **dl_kwargs): + def __init__(self, device: Optional[torch.device] = None, **dl_kwargs: Any) -> None: super().__init__(**dl_kwargs) self._device = device - def __iter__(self): + # TODO: how to type this *angry face" + def __iter__(self): # type: ignore iterator = super().__iter__() if self._device is None: return iterator for item in iterator: yield move_data_to_device(item, self._device) - - -# -# def iterator_wrapper(iter_method: Callable): -# iterator = iter_method() -# for item in iterator: -# print("additional") -# yield item -# -# -# def iterator_decorator(fn): -# def _it(): -# return iterator_wrapper(fn) -# -# return _it -# -# -# if __name__ == "__main__": -# dset = BoringModel().train_dataloader().dataset -# loader = DataLoader(dset, num_workers=2) -# loader.__iter__ = iterator_decorator(loader.__iter__) -# for x in iter(loader): -# print() From 92b105d1b35b075d17943f89f213b6ff47128fbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 16 Oct 2021 14:42:30 +0200 Subject: [PATCH 088/331] add barrier to lite --- pytorch_lightning/lite/lite.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 3b1d9ccc01936..a80ef0bbfb535 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -169,6 +169,9 @@ def print(self, *args: Any, **kwargs: Any) -> None: if self.local_rank == 0: print(*args, **kwargs) + def barrier(self) -> None: + self._strategy.barrier() + def reduce_decision(self, decision: bool) -> bool: return self._strategy.reduce_boolean_decision(decision) From 5ba7333e4802108b21bbd26931a2087cc4aeda47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 16 Oct 2021 14:43:03 +0200 Subject: [PATCH 089/331] redirect attributes and methods on LiteOptimizer --- pytorch_lightning/lite/wrappers.py | 36 ++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 6f719d4e11a06..449047a75ef92 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -34,6 +34,30 @@ def __init__(self, optimizer: Optimizer, accelerator: Accelerator) -> None: def optimizer(self) -> Optimizer: return self._optimizer + @property + def defaults(self): + return self._optimizer.defaults + + @defaults.setter + def defaults(self, defaults): + self._optimizer.defaults = defaults + + @property + def state(self): + return self._optimizer.state + + @state.setter + def state(self, state): + self._optimizer.state = state + + @property + def param_groups(self): + return self._optimizer.param_groups + + @param_groups.setter + def param_groups(self, param_groups): + self._optimizer.param_groups = param_groups + def step(self, closure: Optional[Callable] = None) -> None: self._accelerator.optimizer_step( self._optimizer, @@ -41,6 +65,18 @@ def step(self, closure: Optional[Callable] = None) -> None: model=None, ) + def state_dict(self) -> Dict[str, Any]: + return self._optimizer.state_dict() + + def load_state_dict(self, state_dict: Dict[str, Any]) -> None: + self._optimizer.load_state_dict(state_dict) + + def zero_grad(self, set_to_none: bool = False) -> None: + self._optimizer.zero_grad(set_to_none=set_to_none) + + def add_param_group(self, param_group: Dict[str, Any]) -> None: + self._optimizer.add_param_group(param_group) + class _LiteModule(nn.Module): def __init__(self, module: nn.Module, accelerator: Accelerator) -> None: From 2344d0c19fc8818a99284cd847b0728f3bceb2aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 16 Oct 2021 14:45:43 +0200 Subject: [PATCH 090/331] defaults --- pytorch_lightning/lite/wrappers.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 449047a75ef92..5d8e0fdc5d286 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -34,14 +34,6 @@ def __init__(self, optimizer: Optimizer, accelerator: Accelerator) -> None: def optimizer(self) -> Optimizer: return self._optimizer - @property - def defaults(self): - return self._optimizer.defaults - - @defaults.setter - def defaults(self, defaults): - self._optimizer.defaults = defaults - @property def state(self): return self._optimizer.state From efc0e684b8aeac2ec27d4e06dabad5a2184634e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 16 Oct 2021 14:58:29 +0200 Subject: [PATCH 091/331] fix access to optimizer attributes and methods --- pytorch_lightning/lite/wrappers.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 5d8e0fdc5d286..56240cc6c3c8d 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -11,22 +11,22 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Callable, Optional, Dict, Generator, Iterator, Union +from typing import Any, Callable, Optional import torch from torch import nn as nn, Tensor from torch.optim import Optimizer from torch.utils.data import DataLoader -from torch.utils.data.dataloader import _BaseDataLoaderIter from pytorch_lightning.accelerators import Accelerator from pytorch_lightning.utilities.apply_func import apply_to_collection, move_data_to_device # TODO: add attributes and methods from Optimizer -class _LiteOptimizer(Optimizer): +class _LiteOptimizer: def __init__(self, optimizer: Optimizer, accelerator: Accelerator) -> None: - super().__init__(params=optimizer.param_groups, defaults=getattr(optimizer, "defaults", {})) # type: ignore[call-arg] + self.__dict__ = {k: v for k, v in optimizer.__dict__.items() if k not in ("step", "__del__")} + self.__class__ = type("Lite" + optimizer.__class__.__name__, (self.__class__, optimizer.__class__), {}) self._optimizer = optimizer self._accelerator = accelerator @@ -57,18 +57,6 @@ def step(self, closure: Optional[Callable] = None) -> None: model=None, ) - def state_dict(self) -> Dict[str, Any]: - return self._optimizer.state_dict() - - def load_state_dict(self, state_dict: Dict[str, Any]) -> None: - self._optimizer.load_state_dict(state_dict) - - def zero_grad(self, set_to_none: bool = False) -> None: - self._optimizer.zero_grad(set_to_none=set_to_none) - - def add_param_group(self, param_group: Dict[str, Any]) -> None: - self._optimizer.add_param_group(param_group) - class _LiteModule(nn.Module): def __init__(self, module: nn.Module, accelerator: Accelerator) -> None: From 76f8e2e01aa38b5c1ecc44e87610730c9f7669e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 16 Oct 2021 14:58:52 +0200 Subject: [PATCH 092/331] remove todo --- pytorch_lightning/lite/wrappers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 56240cc6c3c8d..f088010d93ad5 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -22,7 +22,6 @@ from pytorch_lightning.utilities.apply_func import apply_to_collection, move_data_to_device -# TODO: add attributes and methods from Optimizer class _LiteOptimizer: def __init__(self, optimizer: Optimizer, accelerator: Accelerator) -> None: self.__dict__ = {k: v for k, v in optimizer.__dict__.items() if k not in ("step", "__del__")} From 08d336a75061d84f419458fdd464603bb0711940 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 16 Oct 2021 17:06:09 +0200 Subject: [PATCH 093/331] add docs --- pytorch_lightning/lite/lite.py | 112 +++++++++++++++++++++++++++++++-- 1 file changed, 106 insertions(+), 6 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index a80ef0bbfb535..fd7f35f78f66f 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -37,16 +37,31 @@ class LightningLite(ABC): + """Lite accelerates your PyTorch training or inference code with minimal changes required. + + Args: + accelerator: The hardware to run on. Possible choices are: cpu, gpu, tpu. + strategy: Strategy for how to run across multiple devices. Possible choices are: + dp, ddp, ddp_spawn, tpu_spawn, deepspeed, ddp_sharded. + devices: Number of devices to train on (int) or which GPUs to train on (list or str). The value applies + per node. + num_nodes: Number of GPU nodes for distributed training. + precision: Double precision (64), full precision (32), half precision (16) or bfloat16 precision (bf16). + plugins: One or several custom plugins + gpus: Provides the same function as the ``devices`` argument but implies ``accelerator="gpu"``. + tpu_cores: Provides the same function as the ``devices`` argument but implies ``accelerator="tpu"``. + """ + def __init__( self, accelerator: Optional[Union[str, Accelerator]] = None, strategy: Optional[Union[str, TrainingTypePlugin]] = None, - plugins: Optional[Union[PLUGIN_INPUT, List[PLUGIN_INPUT]]] = None, - gpus: Optional[Union[List[int], str, int]] = None, - tpu_cores: Optional[Union[List[int], str, int]] = None, devices: Optional[Union[List[int], str, int]] = None, num_nodes: int = 1, precision: Union[int, str] = 32, + plugins: Optional[Union[PLUGIN_INPUT, List[PLUGIN_INPUT]]] = None, + gpus: Optional[Union[List[int], str, int]] = None, + tpu_cores: Optional[Union[List[int], str, int]] = None, ) -> None: self._check_accelerator_support(accelerator) self._check_strategy_support(strategy) @@ -80,27 +95,37 @@ def __init__( @property def device(self) -> torch.device: + """The current device this process runs on. Use this to create tensors directly on the device if needed.""" return self._accelerator.root_device @property def global_rank(self) -> int: + """The global index of the current process across all devices and nodes.""" return getattr(self._strategy, "global_rank", 0) @property def local_rank(self) -> int: + """The index of the current process among the processes running on the local node.""" return getattr(self._strategy, "local_rank", 0) @property def node_rank(self) -> int: + """The index of the current node.""" return getattr(self._strategy, "node_rank", 0) @property def world_size(self) -> int: + """The total number of processes running across all devices and nodes.""" return getattr(self._strategy, "world_size", 1) @abstractmethod def run(self, *args: Any, **kwargs: Any) -> None: - pass + """All the code inside this run method gets accelerated by Lite. + + Args: + *args: Add any positional arguments you need, e.g., the hyperparameters for your model + **kwargs: Add any keyword arguments you need, e.g., the hyperparameters for your model + """ def setup( self, @@ -108,6 +133,17 @@ def setup( optimizers: Union[Optimizer, Sequence[Optimizer]], move_to_device: bool = True, ) -> Tuple[Union[nn.Module, Sequence[nn.Module]], Union[Optimizer, Sequence[Optimizer]]]: + """Setup models and optimizers for accelerated training. + + Args: + models: A list of models to setup + optimizers: A list of optimizers to setup + move_to_device: If set ``True`` (default), moves the model(s) to the correct device. Set this to ``False`` + and alternatively use :meth:`to_device` manually. + + Returns: + The tuple of wrapped models and optimizers, in the same order they were passed in. + """ # wrap all objects passed in and return them in the same order models = [models] if isinstance(models, nn.Module) else models optimizers = [optimizers] if isinstance(optimizers, Optimizer) else optimizers @@ -123,6 +159,20 @@ def setup( def setup_dataloaders( self, *dataloaders: DataLoader, replace_sampler: bool = True, move_to_device: bool = True ) -> Union[DataLoader, Sequence[DataLoader]]: + """Setup one or multiple dataloaders for accelerated training. If you need different settings for each + dataloader, use :meth:`setup_dataloader` individually. + + Args: + *dataloaders: A single dataloader or a sequence of dataloaders. + replace_sampler: If set ``True`` (default), automatically wraps or replaces the sampler on the dataloader(s) + for distributed training. If you have a custom sampler defined, set this to this argument to ``False``. + move_to_device: If set ``True`` (default), moves the data returned by the dataloader(s) automatially to + the correct device. Set this to ``False`` and alternatively use :meth:`to_device` manually on the + returned data. + + Returns: + The wrapped dataloaders, in the same order they were passed in. + """ # user can call this method independently instead of the general purpose setup method dataloaders = [ self.setup_dataloader(dataloader, replace_sampler=replace_sampler, move_to_device=move_to_device) @@ -134,6 +184,19 @@ def setup_dataloaders( def setup_dataloader( self, dataloader: DataLoader, replace_sampler: bool = True, move_to_device: bool = True ) -> DataLoader: + """Setup a single dataloader for accelerated training. + + Args: + dataloader: The dataloader to accelerate. + replace_sampler: If set ``True`` (default), automatically wraps or replaces the sampler on the dataloader + for distributed training. If you have a custom sampler defined, set this to this argument to ``False``. + move_to_device: If set ``True`` (default), moves the data returned by the dataloader automatially to + the correct device. Set this to ``False`` and alternatively use :meth:`to_device` manually on the + returned data. + + Returns: + The wrapped dataloader. + """ if not replace_sampler or not ( self._requires_distributed_sampler(dataloader) or isinstance(self._accelerator, TPUAccelerator) ): @@ -152,24 +215,61 @@ def setup_dataloader( return _LiteDataLoader(device=device, **kwargs) def backward(self, tensor: Tensor, *args: Any, **kwargs: Any) -> None: - # user will call self.backward(loss) instead of loss.backward() + """Replaces ``loss.backward()`` in your training loop. Handles precision and automatically for you. + + Args: + tensor: The tensor (loss) to back-propagate gradients from. + *args: Optional positional arguments passed to the underlying backward function. + **kwargs: Optional named keyword arguments passed to the underlying backward function. + """ self._accelerator.run_backward(tensor, self._strategy.model, *args, **kwargs) @contextmanager - def forward_context(self) -> Generator[None, None, None]: + def cast(self) -> Generator[None, None, None]: + """A context manager to automatically convert operations for the chosen precision. + + Use this only if the `forward` method of your model does not cover all operations you wish to run with + the chosen precision setting. + """ with self._accelerator.forward_context(): yield def to_device(self, obj: Union[nn.Module, Tensor, Any]) -> Union[nn.Module, Tensor, Any]: + """Move a :class:`torch.nn.Module` or a collection of tensors to the current device, if it is not already + on that device. + + Args: + obj: An object to move to the device. Can be an instance of :class:`torch.nn.Module`, a tensor, or a + (nested) collection of tensors (e.g., a dictionary). + + Returns: + A reference to the object that was moved to the new device. + """ if isinstance(obj, nn.Module): return obj.to(self.device) return move_data_to_device(obj, device=self.device) def print(self, *args: Any, **kwargs: Any) -> None: + """Print something only on the first process. Arguments passed to this method are forwarded to the + Python built-in :func:`print` function.""" if self.local_rank == 0: print(*args, **kwargs) def barrier(self) -> None: + """Wait for all processes to enter this call. Use this to synchronize all parallel processes, but only if + necessary, otherwhise the overhead of synchronization will cause your program to slow down. + + Example:: + + if self.global_rank == 0: + # let process 0 download the dataset + dataset.download_files() + + # let all processes wait before reading the dataset + self.barrier() + + # now all processes can read the files and start training + """ self._strategy.barrier() def reduce_decision(self, decision: bool) -> bool: From a64be1fe057aaf520efa64f0f010cf603e7a33a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 16 Oct 2021 17:06:24 +0200 Subject: [PATCH 094/331] add defaults property --- pytorch_lightning/lite/wrappers.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index f088010d93ad5..94b77e8834d58 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -41,6 +41,14 @@ def state(self): def state(self, state): self._optimizer.state = state + @property + def defaults(self): + return self._optimizer.defaults + + @defaults.setter + def defaults(self, defaults): + self._optimizer.defaults = defaults + @property def param_groups(self): return self._optimizer.param_groups From 2177d13a87f71fca1ab8118c5157750171ffa5eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 16 Oct 2021 17:54:51 +0200 Subject: [PATCH 095/331] update gan debug 64 debug debug --- pl_examples/lite_examples/gan/gan_example.py | 5 ++++- pl_examples/lite_examples/gan/models.py | 1 + pl_examples/lite_examples/gan/run_examples.py | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pl_examples/lite_examples/gan/gan_example.py b/pl_examples/lite_examples/gan/gan_example.py index 976928f3e49ca..0978e9f677767 100644 --- a/pl_examples/lite_examples/gan/gan_example.py +++ b/pl_examples/lite_examples/gan/gan_example.py @@ -54,7 +54,8 @@ class GANTrainer(LightningLite): def run(self): - print("selected plugin: ", self._strategy) + print("strategy: ", self._strategy) + print("precision plugin: ", self._precision_plugin) seed_everything(123) # TODO: how do we handle this in Accelerator? @@ -66,6 +67,7 @@ def run(self): if self.local_rank == 0: dset.MNIST(root=".", download=True) + self.barrier() dataset = dset.MNIST( root=".", transform=transforms.Compose( @@ -106,6 +108,7 @@ def run(self): assert isinstance(optimizerG, _LiteOptimizer) assert isinstance(netG, _LiteModule) + print("parameters dtype", next(netG.parameters()).dtype) for epoch in range(opt.niter): for i, data in enumerate(dataloader, 0): diff --git a/pl_examples/lite_examples/gan/models.py b/pl_examples/lite_examples/gan/models.py index 8f73902ff580c..76f1608bfc5a1 100644 --- a/pl_examples/lite_examples/gan/models.py +++ b/pl_examples/lite_examples/gan/models.py @@ -74,4 +74,5 @@ def __init__(self): def forward(self, input): print("autocast enabled in discriminator: ", torch.is_autocast_enabled()) output = self.main(input) + print("double precision: ", input.dtype == torch.double) return output.view(-1, 1).squeeze(1) diff --git a/pl_examples/lite_examples/gan/run_examples.py b/pl_examples/lite_examples/gan/run_examples.py index 8d77e7f14b7d2..4133fcd3c6338 100644 --- a/pl_examples/lite_examples/gan/run_examples.py +++ b/pl_examples/lite_examples/gan/run_examples.py @@ -5,6 +5,7 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--accelerator", type=str, default=None) + parser.add_argument("--strategy", type=str, default=None) parser.add_argument("--gpus", type=int, default=None) parser.add_argument("--devices", type=int, default=1) parser.add_argument("--precision", type=int, default=32) From 672f8f163c247f1b550ae057fa61ab726410c3cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sun, 17 Oct 2021 15:42:04 +0200 Subject: [PATCH 096/331] delete unused example --- .../gan/deepspeed_gan_example.py | 176 ------------------ 1 file changed, 176 deletions(-) delete mode 100644 pl_examples/lite_examples/gan/deepspeed_gan_example.py diff --git a/pl_examples/lite_examples/gan/deepspeed_gan_example.py b/pl_examples/lite_examples/gan/deepspeed_gan_example.py deleted file mode 100644 index 2e4281c969ec9..0000000000000 --- a/pl_examples/lite_examples/gan/deepspeed_gan_example.py +++ /dev/null @@ -1,176 +0,0 @@ -""" -DCGAN - Adapted from pytorch/examples - -Launch it with this command: - -python -m torch.distributed.run --nproc_per_node=2 gan_example.py - -""" -from __future__ import print_function - -import argparse -import os -import random - -import torch -import torch.nn as nn -import torch.nn.parallel -import torch.optim as optim -import torch.utils.data -import torchvision.datasets as dset -import torchvision.transforms as transforms -import torchvision.utils as vutils - -from pl_examples.lite_examples.gan.models import Discriminator, Generator, weights_init -from pytorch_lightning import seed_everything -from pytorch_lightning.lite import LightningLite -from pytorch_lightning.lite.wrappers import _LiteOptimizer, _LiteModule - -parser = argparse.ArgumentParser() -parser.add_argument("--workers", type=int, help="number of data loading workers", default=0) -parser.add_argument("--batchSize", type=int, default=64, help="input batch size") -parser.add_argument( - "--imageSize", - type=int, - default=64, - help="the height / width of the input image to network", -) -parser.add_argument("--niter", type=int, default=25, help="number of epochs to train for") -parser.add_argument("--lr", type=float, default=0.0002, help="learning rate, default=0.0002") -parser.add_argument("--beta1", type=float, default=0.5, help="beta1 for adam. default=0.5") -parser.add_argument("--ngpu", type=int, default=1, help="number of GPUs to use") -parser.add_argument("--netG", default="", help="path to netG (to continue training)") -parser.add_argument("--netD", default="", help="path to netD (to continue training)") -parser.add_argument("--outf", default="./lightning_logs", help="folder to output images and model checkpoints") -parser.add_argument("--local_rank", type=int, default=0) - -opt, _ = parser.parse_known_args() -os.makedirs(opt.outf, exist_ok=True) -ngpu = int(opt.ngpu) - -nz = 100 - - -class GANTrainer(LightningLite): - def run(self): - print("selected plugin: ", self._strategy) - seed_everything(123) - - # TODO: how do we handle this in Accelerator? - # torch.cuda.set_device(opt.local_rank) - # TODO: how do we handle this? - # os.environ["LOCAL_RANK"] = str(opt.local_rank) - # os.environ["NODE_RANK"] = str(opt.local_rank) - - if self.local_rank == 0: - dset.MNIST(root=".", download=True) - - dataset = dset.MNIST( - root=".", - transform=transforms.Compose( - [ - transforms.Resize(opt.imageSize), - transforms.ToTensor(), - transforms.Normalize((0.5,), (0.5,)), - ] - ), - ) - dataloader = torch.utils.data.DataLoader( - dataset, batch_size=opt.batchSize, shuffle=True, num_workers=opt.workers - ) - - dataloader = self.setup_dataloader(dataloader) - # assert isinstance(dataloader.sampler, DistributedSampler) - - netG = Generator() - netG.apply(weights_init) - - netD = Discriminator() - netD.apply(weights_init) - - self.to_device(netG) - self.to_device(netD) - - criterion = nn.BCELoss() - - fixed_noise = torch.randn(opt.batchSize, nz, 1, 1, device=self.device) - real_label = 1 - fake_label = 0 - - # setup optimizer - optimizerD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) - optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) - - (netG, netD), (optimizerG, optimizerD) = self.setup(models=(netG, netD), optimizers=(optimizerG, optimizerD)) - - assert isinstance(optimizerG, _LiteOptimizer) - assert isinstance(netG, _LiteModule) - - for epoch in range(opt.niter): - for i, data in enumerate(dataloader, 0): - ############################ - # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z))) - ########################### - # train with real - netD.zero_grad() - real_cpu = self.to_device(data[0]) - batch_size = real_cpu.size(0) - label = torch.full((batch_size,), real_label, dtype=real_cpu.dtype, device=self.device) - - output = netD(real_cpu) - errD_real = criterion(output, label) - netD.backward(errD_real) - D_x = output.mean().item() - - # train with fake - noise = torch.randn(batch_size, nz, 1, 1, device=self.device) - fake = netG(noise) - label.fill_(fake_label) - output = netD(fake.detach()) - errD_fake = criterion(output, label) - netD.backward(errD_fake) - D_G_z1 = output.mean().item() - errD = errD_real + errD_fake - netD.step() - - ############################ - # (2) Update G network: maximize log(D(G(z))) - ########################### - netG.zero_grad() - label.fill_(real_label) # fake labels are real for generator cost - output = netD(fake) - errG = criterion(output, label) - netG.backward(errG) - D_G_z2 = output.mean().item() - netG.step() - - print( - "[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)): %.4f / %.4f" - % ( - epoch, - opt.niter, - i, - len(dataloader), - errD.item(), - errG.item(), - D_x, - D_G_z1, - D_G_z2, - ) - ) - if i % 100 == 0: - vutils.save_image(real_cpu, "%s/real_samples.png" % opt.outf, normalize=True) - fake = netG(fixed_noise) - vutils.save_image( - fake.detach(), - "%s/fake_samples_epoch_%03d.png" % (opt.outf, epoch), - normalize=True, - ) - # do checkpointing - # torch.save(netG.state_dict(), "%s/netG_epoch_%d.pth" % (opt.outf, epoch)) - # torch.save(netD.state_dict(), "%s/netD_epoch_%d.pth" % (opt.outf, epoch)) - - -if __name__ == "__main__": - gan = GANTrainer(gpus=1, strategy="deepspeed", accelerator="gpu") - gan.run() From 0d08f9340cbe8ca864c24e27dd85a9d060619c7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sun, 17 Oct 2021 15:42:39 +0200 Subject: [PATCH 097/331] restrict one model per setup call --- pl_examples/lite_examples/gan/gan_example.py | 3 +- pytorch_lightning/lite/lite.py | 36 +++++++++---------- .../plugins/training_type/deepspeed.py | 4 +-- .../plugins/training_type/sharded.py | 12 +++---- .../training_type/training_type_plugin.py | 6 ++-- 5 files changed, 29 insertions(+), 32 deletions(-) diff --git a/pl_examples/lite_examples/gan/gan_example.py b/pl_examples/lite_examples/gan/gan_example.py index 0978e9f677767..bad8abc7903ba 100644 --- a/pl_examples/lite_examples/gan/gan_example.py +++ b/pl_examples/lite_examples/gan/gan_example.py @@ -104,7 +104,8 @@ def run(self): optimizerD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) - (netG, netD), (optimizerG, optimizerD) = self.setup(models=(netG, netD), optimizers=(optimizerG, optimizerD)) + netG, optimizerG = self.setup(netG, optimizerG) + netD, optimizerD = self.setup(netD, optimizerD) assert isinstance(optimizerG, _LiteOptimizer) assert isinstance(netG, _LiteModule) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index fd7f35f78f66f..a2ccdd9ed60e4 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -129,32 +129,30 @@ def run(self, *args: Any, **kwargs: Any) -> None: def setup( self, - models: Union[nn.Module, Sequence[nn.Module]], - optimizers: Union[Optimizer, Sequence[Optimizer]], + model: nn.Module, + optimizers: Union[Optimizer, List[Optimizer]], move_to_device: bool = True, - ) -> Tuple[Union[nn.Module, Sequence[nn.Module]], Union[Optimizer, Sequence[Optimizer]]]: - """Setup models and optimizers for accelerated training. + ) -> Tuple[nn.Module, Union[_LiteOptimizer, List[_LiteOptimizer]]]: + """Setup a model and its optimizers for accelerated training. Args: - models: A list of models to setup + model: A model to setup optimizers: A list of optimizers to setup - move_to_device: If set ``True`` (default), moves the model(s) to the correct device. Set this to ``False`` + move_to_device: If set ``True`` (default), moves the model to the correct device. Set this to ``False`` and alternatively use :meth:`to_device` manually. Returns: - The tuple of wrapped models and optimizers, in the same order they were passed in. + The tuple of the wrapped model and list of optimizers, in the same order they were passed in. """ # wrap all objects passed in and return them in the same order - models = [models] if isinstance(models, nn.Module) else models optimizers = [optimizers] if isinstance(optimizers, Optimizer) else optimizers - models, optimizers = self._setup_models_and_optimizers(models, optimizers) + model, optimizers = self._setup_model_and_optimizers(model, optimizers) if move_to_device: - models = [self.to_device(model) for model in models] + model = self.to_device(model) - models = models[0] if len(models) == 1 else models optimizers = optimizers[0] if len(optimizers) == 1 else optimizers - return models, optimizers + return model, optimizers def setup_dataloaders( self, *dataloaders: DataLoader, replace_sampler: bool = True, move_to_device: bool = True @@ -293,16 +291,16 @@ def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> None: run_method(*args, **kwargs) # TODO: any teardown needed here? - def _setup_models_and_optimizers( + def _setup_model_and_optimizers( self, - models: Sequence[nn.Module], - optimizers: Sequence[Optimizer], - ) -> Tuple[Sequence[_LiteModule], Sequence[_LiteOptimizer]]: + model: nn.Module, + optimizers: Union[Optimizer, List[Optimizer]], + ) -> Tuple[_LiteModule, Union[_LiteOptimizer, List[_LiteOptimizer]]]: # Let accelerator/plugin wrap and connect the models and optimizers - models, optimizers = self._strategy.setup_models_and_optimizers(models, optimizers) - models = [_LiteModule(module=model, accelerator=self._accelerator) for model in models] + [model], optimizers = self._strategy.setup_models_and_optimizers([model], optimizers) + model = _LiteModule(module=model, accelerator=self._accelerator) optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self._accelerator) for optimizer in optimizers] - return models, optimizers + return model, optimizers def _requires_distributed_sampler(self, dataloader: DataLoader) -> bool: return ( diff --git a/pytorch_lightning/plugins/training_type/deepspeed.py b/pytorch_lightning/plugins/training_type/deepspeed.py index 8e1d9fcc1f3e2..09814260ed3c4 100644 --- a/pytorch_lightning/plugins/training_type/deepspeed.py +++ b/pytorch_lightning/plugins/training_type/deepspeed.py @@ -382,8 +382,8 @@ def pre_dispatch(self): # TODO: avoid code duplication by letting the plugin reuse this method def setup_models_and_optimizers( - self, models: Sequence[Module], optimizers: Sequence[Optimizer] - ) -> Tuple[Sequence[Module], Sequence[Optimizer]]: + self, models: List[Module], optimizers: List[Optimizer] + ) -> Tuple[List[Module], List[Optimizer]]: if not (len(models) == len(optimizers) == 1): raise ValueError( f"Currently only one model and one optimizer is supported with DeepSpeed." diff --git a/pytorch_lightning/plugins/training_type/sharded.py b/pytorch_lightning/plugins/training_type/sharded.py index fd24cfef2a327..670edcfb096cf 100644 --- a/pytorch_lightning/plugins/training_type/sharded.py +++ b/pytorch_lightning/plugins/training_type/sharded.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from contextlib import contextmanager -from typing import Dict, Generator, Optional, Sequence, Tuple, List, Union +from typing import Dict, Generator, Optional, Tuple, List, Union import torch from torch.nn import Module @@ -38,8 +38,8 @@ class DDPShardedPlugin(DDPPlugin): _REDUCE_BUFFER_SIZE_DEFAULT = 2 ** 23 # 8M def setup_models_and_optimizers( - self, models: Sequence[Module], optimizers: Sequence[Optimizer] - ) -> Tuple[Sequence[Module], Sequence[Optimizer]]: + self, models: List[Module], optimizers: List[Optimizer] + ) -> Tuple[List[Module], List[Optimizer]]: if len(models) > 1: raise ValueError( f"DDPSharded only supports a single model with one or several optimizers. Got {len(models)} models." @@ -63,9 +63,7 @@ def configure_ddp(self) -> None: trainer.optimizers = optimizers trainer.convert_to_lightning_optimizers() - def _reinit_optimizers_with_oss( - self, optimizers: Sequence[Union[Optimizer, LightningOptimizer]] - ) -> Sequence["OSS"]: + def _reinit_optimizers_with_oss(self, optimizers: List[Union[Optimizer, LightningOptimizer]]) -> List["OSS"]: for x, optimizer in enumerate(optimizers): if isinstance(optimizer, LightningOptimizer): optimizer = optimizer._optimizer @@ -85,7 +83,7 @@ def _reinit_optimizers_with_oss( del optimizer return optimizers - def _wrap_optimizers(self, optimizers: Sequence[Optimizer]) -> Sequence["OSS"]: + def _wrap_optimizers(self, optimizers: List[Optimizer]) -> List["OSS"]: if self.model is not None and self.model.trainer.state.fn != TrainerFn.FITTING: return optimizers diff --git a/pytorch_lightning/plugins/training_type/training_type_plugin.py b/pytorch_lightning/plugins/training_type/training_type_plugin.py index bc0dd7218e176..7782e3ea7ab40 100644 --- a/pytorch_lightning/plugins/training_type/training_type_plugin.py +++ b/pytorch_lightning/plugins/training_type/training_type_plugin.py @@ -13,7 +13,7 @@ # limitations under the License. import contextlib from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, Generator, Iterable, Mapping, Optional, Sequence, Union, Tuple +from typing import Any, Callable, Dict, Generator, Iterable, Mapping, Optional, Sequence, Union, Tuple, List import torch from torch import Tensor @@ -66,8 +66,8 @@ def setup_dataloader(self, dataloader: DataLoader) -> DataLoader: return dataloader def setup_models_and_optimizers( - self, models: Sequence[Module], optimizers: Sequence[Optimizer] - ) -> Tuple[Sequence[Module], Sequence[Optimizer]]: + self, models: List[Module], optimizers: List[Optimizer] + ) -> Tuple[List[Module], List[Optimizer]]: models = [self.setup_model(model) for model in models] optimizers = [self.setup_optimizer(optimizer) for optimizer in optimizers] return models, optimizers From 2a3e32da46c4a58a72e9339c1cb0bad260254737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sun, 17 Oct 2021 15:48:28 +0200 Subject: [PATCH 098/331] mark setup_dataloader protected --- pl_examples/lite_examples/gan/gan_example.py | 2 +- pytorch_lightning/lite/lite.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pl_examples/lite_examples/gan/gan_example.py b/pl_examples/lite_examples/gan/gan_example.py index bad8abc7903ba..67d15eda98126 100644 --- a/pl_examples/lite_examples/gan/gan_example.py +++ b/pl_examples/lite_examples/gan/gan_example.py @@ -82,7 +82,7 @@ def run(self): dataset, batch_size=opt.batchSize, shuffle=True, num_workers=opt.workers ) - dataloader = self.setup_dataloader(dataloader) + dataloader = self.setup_dataloaders(dataloader) # assert isinstance(dataloader.sampler, DistributedSampler) netG = Generator() diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index a2ccdd9ed60e4..c192d57e2991f 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -156,9 +156,9 @@ def setup( def setup_dataloaders( self, *dataloaders: DataLoader, replace_sampler: bool = True, move_to_device: bool = True - ) -> Union[DataLoader, Sequence[DataLoader]]: + ) -> Union[DataLoader, List[DataLoader]]: """Setup one or multiple dataloaders for accelerated training. If you need different settings for each - dataloader, use :meth:`setup_dataloader` individually. + dataloader, call this method individually for each one. Args: *dataloaders: A single dataloader or a sequence of dataloaders. @@ -173,13 +173,13 @@ def setup_dataloaders( """ # user can call this method independently instead of the general purpose setup method dataloaders = [ - self.setup_dataloader(dataloader, replace_sampler=replace_sampler, move_to_device=move_to_device) + self._setup_dataloader(dataloader, replace_sampler=replace_sampler, move_to_device=move_to_device) for dataloader in dataloaders ] dataloaders = dataloaders[0] if len(dataloaders) == 1 else dataloaders return dataloaders - def setup_dataloader( + def _setup_dataloader( self, dataloader: DataLoader, replace_sampler: bool = True, move_to_device: bool = True ) -> DataLoader: """Setup a single dataloader for accelerated training. From 3ad693c698a47bb1113856822b59893dd4b96014 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sun, 17 Oct 2021 16:17:38 +0200 Subject: [PATCH 099/331] add feature teaser --- pytorch_lightning/lite/lite.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index c192d57e2991f..76d5a6607c8e2 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -39,6 +39,13 @@ class LightningLite(ABC): """Lite accelerates your PyTorch training or inference code with minimal changes required. + - Automatic placement of models and data onto the device + - Automatic support for mixed and double precision (smaller memory footprint) + - Seamless switching between hardware (CPU, GPU, TPU) and distributed training strategies + (data-parallel training, sharded training, etc.) + - Automated spawning of processes, no launch utilities required + - Multi-node support + Args: accelerator: The hardware to run on. Possible choices are: cpu, gpu, tpu. strategy: Strategy for how to run across multiple devices. Possible choices are: From 82daab9cdde8909eec92ce2c5e18f47304e030fa Mon Sep 17 00:00:00 2001 From: Kaushik B Date: Mon, 18 Oct 2021 14:45:25 +0530 Subject: [PATCH 100/331] Fix(tpu): remove optimizer_idx from run_optimizer_step signature --- pytorch_lightning/accelerators/tpu.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pytorch_lightning/accelerators/tpu.py b/pytorch_lightning/accelerators/tpu.py index 68925ab67aca9..bd47fd11a5591 100644 --- a/pytorch_lightning/accelerators/tpu.py +++ b/pytorch_lightning/accelerators/tpu.py @@ -49,9 +49,7 @@ def setup(self, trainer: "pl.Trainer") -> None: raise MisconfigurationException("TPUs only support a single tpu core or tpu spawn training.") return super().setup(trainer) - def run_optimizer_step( - self, optimizer: Optimizer, optimizer_idx: int, lambda_closure: Callable, **kwargs: Any - ) -> None: + def run_optimizer_step(self, optimizer: Optimizer, lambda_closure: Callable, **kwargs: Any) -> None: xm.optimizer_step(optimizer, optimizer_args={"closure": lambda_closure, **kwargs}) def _move_optimizer_state(self, device: Optional[torch.device] = None) -> None: From 88fdd5026a48c838ac84d38a6260678a8a5d9ea5 Mon Sep 17 00:00:00 2001 From: Kaushik B Date: Mon, 18 Oct 2021 14:53:22 +0530 Subject: [PATCH 101/331] Fix deepspeed imports --- pytorch_lightning/plugins/precision/deepspeed_precision.py | 1 - pytorch_lightning/plugins/training_type/deepspeed.py | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/plugins/precision/deepspeed_precision.py b/pytorch_lightning/plugins/precision/deepspeed_precision.py index 4f45d8ec9912c..7fbd3f4056bda 100644 --- a/pytorch_lightning/plugins/precision/deepspeed_precision.py +++ b/pytorch_lightning/plugins/precision/deepspeed_precision.py @@ -13,7 +13,6 @@ # limitations under the License. from typing import Any, Callable, Optional, Union -import deepspeed from torch import Tensor from torch.nn import Module from torch.optim import Optimizer diff --git a/pytorch_lightning/plugins/training_type/deepspeed.py b/pytorch_lightning/plugins/training_type/deepspeed.py index 09814260ed3c4..07ea169f7d5ff 100644 --- a/pytorch_lightning/plugins/training_type/deepspeed.py +++ b/pytorch_lightning/plugins/training_type/deepspeed.py @@ -19,10 +19,9 @@ import platform from collections import OrderedDict from pathlib import Path -from typing import Any, Callable, Dict, Generator, List, Mapping, Optional, Tuple, Union, Sequence +from typing import Any, Callable, Dict, Generator, List, Mapping, Optional, Tuple, Union import torch -from deepspeed import DeepSpeedEngine from torch.nn import Module from torch.optim import Optimizer @@ -399,7 +398,7 @@ def setup_models_and_optimizers( # self._set_deepspeed_activation_checkpointing() return [self._model], [optimizer] - def _setup_model_and_optimizer(self, model: Module, optimizer: Optimizer) -> Tuple[DeepSpeedEngine, Optimizer]: + def _setup_model_and_optimizer(self, model: Module, optimizer: Optimizer): # TODO: shouldn't this be optimizer.parameters? model_parameters = filter(lambda p: p.requires_grad, model.parameters()) deepspeed_engine, deepspeed_optimizer, _, _ = deepspeed.initialize( From 0b6f679bc567987a80c123e0e56f00d90b4c673b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 18 Oct 2021 11:34:48 +0200 Subject: [PATCH 102/331] update type for setup --- pytorch_lightning/lite/lite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 76d5a6607c8e2..1d17d44e9765a 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -301,8 +301,8 @@ def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> None: def _setup_model_and_optimizers( self, model: nn.Module, - optimizers: Union[Optimizer, List[Optimizer]], - ) -> Tuple[_LiteModule, Union[_LiteOptimizer, List[_LiteOptimizer]]]: + optimizers: List[Optimizer], + ) -> Tuple[_LiteModule, List[_LiteOptimizer]]: # Let accelerator/plugin wrap and connect the models and optimizers [model], optimizers = self._strategy.setup_models_and_optimizers([model], optimizers) model = _LiteModule(module=model, accelerator=self._accelerator) From 6ed19461e3e0dd9dba5c64377d90e641b70a4a13 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Oct 2021 10:38:15 +0000 Subject: [PATCH 103/331] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pl_examples/lite_examples/gan/gan_example.py | 3 +-- pl_examples/lite_examples/gan/models.py | 4 +-- .../lite_examples/simple/mnist_example.py | 5 ++-- pytorch_lightning/accelerators/accelerator.py | 2 +- pytorch_lightning/lite/lite.py | 27 +++++++++++-------- pytorch_lightning/lite/wrappers.py | 3 ++- .../plugins/precision/native_amp.py | 4 +-- .../plugins/precision/precision_plugin.py | 2 +- .../plugins/training_type/ddp_spawn.py | 2 +- .../plugins/training_type/sharded.py | 2 +- .../training_type/training_type_plugin.py | 9 ++++--- 11 files changed, 36 insertions(+), 27 deletions(-) diff --git a/pl_examples/lite_examples/gan/gan_example.py b/pl_examples/lite_examples/gan/gan_example.py index 67d15eda98126..9cf4a91ed3ff3 100644 --- a/pl_examples/lite_examples/gan/gan_example.py +++ b/pl_examples/lite_examples/gan/gan_example.py @@ -6,7 +6,6 @@ python -m torch.distributed.run --nproc_per_node=2 gan_example.py """ -from __future__ import print_function import argparse import os @@ -25,7 +24,7 @@ from pl_examples.lite_examples.gan.models import Discriminator, Generator, weights_init from pytorch_lightning import seed_everything from pytorch_lightning.lite import LightningLite -from pytorch_lightning.lite.wrappers import _LiteOptimizer, _LiteModule +from pytorch_lightning.lite.wrappers import _LiteModule, _LiteOptimizer parser = argparse.ArgumentParser() parser.add_argument("--workers", type=int, help="number of data loading workers", default=0) diff --git a/pl_examples/lite_examples/gan/models.py b/pl_examples/lite_examples/gan/models.py index 76f1608bfc5a1..5ccdec18aebc2 100644 --- a/pl_examples/lite_examples/gan/models.py +++ b/pl_examples/lite_examples/gan/models.py @@ -18,7 +18,7 @@ def weights_init(m): class Generator(nn.Module): def __init__(self): - super(Generator, self).__init__() + super().__init__() self.main = nn.Sequential( # input is Z, going into a convolution nn.ConvTranspose2d(nz, ngf * 8, 4, 1, 0, bias=False), @@ -49,7 +49,7 @@ def forward(self, input): class Discriminator(nn.Module): def __init__(self): - super(Discriminator, self).__init__() + super().__init__() self.main = nn.Sequential( # input is (nc) x 64 x 64 nn.Conv2d(nc, ndf, 4, 2, 1, bias=False), diff --git a/pl_examples/lite_examples/simple/mnist_example.py b/pl_examples/lite_examples/simple/mnist_example.py index c0a1931280891..1d9cb715c4137 100644 --- a/pl_examples/lite_examples/simple/mnist_example.py +++ b/pl_examples/lite_examples/simple/mnist_example.py @@ -1,11 +1,12 @@ import argparse + import torch import torch.nn as nn import torch.nn.functional as F import torch.optim as optim +from torch.optim.lr_scheduler import StepLR from torch.utils.data import DistributedSampler from torchvision import datasets, transforms -from torch.optim.lr_scheduler import StepLR from pytorch_lightning import seed_everything from pytorch_lightning.lite import LightningLite @@ -13,7 +14,7 @@ class Net(nn.Module): def __init__(self): - super(Net, self).__init__() + super().__init__() self.conv1 = nn.Conv2d(1, 32, 3, 1) self.conv2 = nn.Conv2d(32, 64, 3, 1) self.dropout1 = nn.Dropout(0.25) diff --git a/pytorch_lightning/accelerators/accelerator.py b/pytorch_lightning/accelerators/accelerator.py index edd80e2747813..691b117185b0d 100644 --- a/pytorch_lightning/accelerators/accelerator.py +++ b/pytorch_lightning/accelerators/accelerator.py @@ -315,7 +315,7 @@ def backward(self, closure_loss: Tensor, *args: Any, **kwargs: Any) -> Tensor: return closure_loss def run_backward(self, tensor: Tensor, model, *args, **kwargs) -> None: - """Lightning-independent backward logic""" + """Lightning-independent backward logic.""" self.precision_plugin.run_backward(tensor, model, *args, **kwargs) def optimizer_step( diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 1d17d44e9765a..aa8ea606cfc5e 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -12,26 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -from abc import abstractmethod, ABC +from abc import ABC, abstractmethod from collections import Callable from contextlib import contextmanager from functools import partial from pathlib import Path -from typing import Any, Optional, Sequence, Union, List, Dict, Tuple, Generator +from typing import Any, Dict, Generator, List, Optional, Sequence, Tuple, Union import torch import torch.nn as nn from torch import Tensor from torch.optim import Optimizer -from torch.utils.data import DataLoader, DistributedSampler, SequentialSampler, RandomSampler, Sampler +from torch.utils.data import DataLoader, DistributedSampler, RandomSampler, Sampler, SequentialSampler from pytorch_lightning import Trainer from pytorch_lightning.accelerators import Accelerator, TPUAccelerator -from pytorch_lightning.lite.wrappers import _LiteOptimizer, _LiteModule, _LiteDataLoader -from pytorch_lightning.plugins import PLUGIN_INPUT, DDPSpawnPlugin, TrainingTypePlugin, DeepSpeedPlugin +from pytorch_lightning.lite.wrappers import _LiteDataLoader, _LiteModule, _LiteOptimizer +from pytorch_lightning.plugins import DDPSpawnPlugin, DeepSpeedPlugin, PLUGIN_INPUT, TrainingTypePlugin from pytorch_lightning.trainer.connectors.accelerator_connector import AcceleratorConnector from pytorch_lightning.trainer.data_loading import TrainerDataLoadingMixin -from pytorch_lightning.utilities import move_data_to_device, DistributedType, DeviceType +from pytorch_lightning.utilities import DeviceType, DistributedType, move_data_to_device from pytorch_lightning.utilities.data import has_iterable_dataset from pytorch_lightning.utilities.exceptions import MisconfigurationException @@ -102,7 +102,10 @@ def __init__( @property def device(self) -> torch.device: - """The current device this process runs on. Use this to create tensors directly on the device if needed.""" + """The current device this process runs on. + + Use this to create tensors directly on the device if needed. + """ return self._accelerator.root_device @property @@ -233,8 +236,8 @@ def backward(self, tensor: Tensor, *args: Any, **kwargs: Any) -> None: def cast(self) -> Generator[None, None, None]: """A context manager to automatically convert operations for the chosen precision. - Use this only if the `forward` method of your model does not cover all operations you wish to run with - the chosen precision setting. + Use this only if the `forward` method of your model does not cover all operations you wish to run with the + chosen precision setting. """ with self._accelerator.forward_context(): yield @@ -255,8 +258,10 @@ def to_device(self, obj: Union[nn.Module, Tensor, Any]) -> Union[nn.Module, Tens return move_data_to_device(obj, device=self.device) def print(self, *args: Any, **kwargs: Any) -> None: - """Print something only on the first process. Arguments passed to this method are forwarded to the - Python built-in :func:`print` function.""" + """Print something only on the first process. + + Arguments passed to this method are forwarded to the Python built-in :func:`print` function. + """ if self.local_rank == 0: print(*args, **kwargs) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 94b77e8834d58..7a332952ee15e 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -14,7 +14,8 @@ from typing import Any, Callable, Optional import torch -from torch import nn as nn, Tensor +from torch import nn as nn +from torch import Tensor from torch.optim import Optimizer from torch.utils.data import DataLoader diff --git a/pytorch_lightning/plugins/precision/native_amp.py b/pytorch_lightning/plugins/precision/native_amp.py index 27f0fcbccf4c5..97aa134f5f99b 100644 --- a/pytorch_lightning/plugins/precision/native_amp.py +++ b/pytorch_lightning/plugins/precision/native_amp.py @@ -100,7 +100,7 @@ def pre_optimizer_step( return False def post_optimizer_step(self, optimizer: "Optimizer", optimizer_idx: int) -> None: - """Updates the GradScaler""" + """Updates the GradScaler.""" self.run_post_optimizer_step(optimizer) def run_pre_optimizer_step(self, optimizer: "Optimizer") -> None: @@ -119,7 +119,7 @@ def autocast_context_manager(self) -> torch.cuda.amp.autocast: @contextmanager def forward_context(self) -> Generator[None, None, None]: - """Enable autocast context""" + """Enable autocast context.""" with self.autocast_context_manager(): yield diff --git a/pytorch_lightning/plugins/precision/precision_plugin.py b/pytorch_lightning/plugins/precision/precision_plugin.py index e06f7ac7ccc99..fd7ee03f7154d 100644 --- a/pytorch_lightning/plugins/precision/precision_plugin.py +++ b/pytorch_lightning/plugins/precision/precision_plugin.py @@ -158,7 +158,7 @@ def post_dispatch(self) -> None: @contextlib.contextmanager def forward_context(self) -> Generator: - """A contextmanager for managing model forward/training_step/evaluation_step/predict_step""" + """A contextmanager for managing model forward/training_step/evaluation_step/predict_step.""" yield @contextlib.contextmanager diff --git a/pytorch_lightning/plugins/training_type/ddp_spawn.py b/pytorch_lightning/plugins/training_type/ddp_spawn.py index bfa1c51867fb7..07fa4e455df26 100644 --- a/pytorch_lightning/plugins/training_type/ddp_spawn.py +++ b/pytorch_lightning/plugins/training_type/ddp_spawn.py @@ -16,7 +16,7 @@ import re from functools import partial from multiprocessing.queues import SimpleQueue -from typing import Any, Dict, List, Optional, Union, Callable +from typing import Any, Callable, Dict, List, Optional, Union import numpy as np import torch diff --git a/pytorch_lightning/plugins/training_type/sharded.py b/pytorch_lightning/plugins/training_type/sharded.py index 670edcfb096cf..2b194fe462cf4 100644 --- a/pytorch_lightning/plugins/training_type/sharded.py +++ b/pytorch_lightning/plugins/training_type/sharded.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from contextlib import contextmanager -from typing import Dict, Generator, Optional, Tuple, List, Union +from typing import Dict, Generator, List, Optional, Tuple, Union import torch from torch.nn import Module diff --git a/pytorch_lightning/plugins/training_type/training_type_plugin.py b/pytorch_lightning/plugins/training_type/training_type_plugin.py index 230c341f5213d..092a46c79531f 100644 --- a/pytorch_lightning/plugins/training_type/training_type_plugin.py +++ b/pytorch_lightning/plugins/training_type/training_type_plugin.py @@ -13,7 +13,7 @@ # limitations under the License. import contextlib from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, Generator, Iterable, Mapping, Optional, Sequence, Union, Tuple, List +from typing import Any, Callable, Dict, Generator, Iterable, List, Mapping, Optional, Sequence, Tuple, Union import torch from torch import Tensor @@ -62,7 +62,10 @@ def setup(self) -> None: """Called by the accelerator to finish setup.""" def setup_dataloader(self, dataloader: DataLoader) -> DataLoader: - """Called by the accelerator. The plugin wraps and modifies the dataloader as needed.""" + """Called by the accelerator. + + The plugin wraps and modifies the dataloader as needed. + """ return dataloader def setup_models_and_optimizers( @@ -306,7 +309,7 @@ def remove_checkpoint(self, filepath: _PATH) -> None: @contextlib.contextmanager def forward_context(self) -> Generator: - """A contextmanager for managing model forward/training_step/evaluation_step/predict_step""" + """A contextmanager for managing model forward/training_step/evaluation_step/predict_step.""" yield @contextlib.contextmanager From f0d9452f177b6af7f621a2701c8166666bb75c73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 18 Oct 2021 13:01:57 +0200 Subject: [PATCH 104/331] remove unused setup method --- pytorch_lightning/plugins/training_type/ddp.py | 6 ------ .../plugins/training_type/training_type_plugin.py | 7 ------- 2 files changed, 13 deletions(-) diff --git a/pytorch_lightning/plugins/training_type/ddp.py b/pytorch_lightning/plugins/training_type/ddp.py index 6e10ec93495f8..7fc53b2b05b3d 100644 --- a/pytorch_lightning/plugins/training_type/ddp.py +++ b/pytorch_lightning/plugins/training_type/ddp.py @@ -191,12 +191,6 @@ def setup_model(self, model: Module) -> Module: ) return model - def setup_dataloader(self, dataloader: DataLoader) -> DataLoader: - kwargs = self.distributed_sampler_kwargs - sampler = DistributedSampler(dataloader.dataset, **kwargs) - # dataloader = replace_sampler(dataloader, sampler) - return dataloader - def _call_children_scripts(self): # bookkeeping of spawned processes self._check_can_spawn_children() diff --git a/pytorch_lightning/plugins/training_type/training_type_plugin.py b/pytorch_lightning/plugins/training_type/training_type_plugin.py index 092a46c79531f..64a1ed6b32c2e 100644 --- a/pytorch_lightning/plugins/training_type/training_type_plugin.py +++ b/pytorch_lightning/plugins/training_type/training_type_plugin.py @@ -61,13 +61,6 @@ def setup_environment(self) -> None: def setup(self) -> None: """Called by the accelerator to finish setup.""" - def setup_dataloader(self, dataloader: DataLoader) -> DataLoader: - """Called by the accelerator. - - The plugin wraps and modifies the dataloader as needed. - """ - return dataloader - def setup_models_and_optimizers( self, models: List[Module], optimizers: List[Optimizer] ) -> Tuple[List[Module], List[Optimizer]]: From 06d04095e52d3ad3678f36f9ceec3dd726f98b67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 18 Oct 2021 13:16:39 +0200 Subject: [PATCH 105/331] remove unused forward_context() usages --- pytorch_lightning/accelerators/accelerator.py | 13 ++++--------- pytorch_lightning/lite/lite.py | 2 +- .../plugins/training_type/training_type_plugin.py | 5 ----- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/pytorch_lightning/accelerators/accelerator.py b/pytorch_lightning/accelerators/accelerator.py index 691b117185b0d..2771a3813bce0 100644 --- a/pytorch_lightning/accelerators/accelerator.py +++ b/pytorch_lightning/accelerators/accelerator.py @@ -211,7 +211,7 @@ def training_step(self, step_kwargs: Dict[str, Union[Any, int]]) -> STEP_OUTPUT: See :meth:`~pytorch_lightning.core.lightning.LightningModule.training_step` for more details """ - with self.precision_plugin.forward_context(), self.training_type_plugin.forward_context(): + with self.precision_plugin.forward_context(): return self.training_type_plugin.training_step(*step_kwargs.values()) def post_training_step(self) -> None: @@ -231,7 +231,7 @@ def validation_step(self, step_kwargs: Dict[str, Union[Any, int]]) -> Optional[S See :meth:`~pytorch_lightning.core.lightning.LightningModule.validation_step` for more details """ - with self.precision_plugin.forward_context(), self.training_type_plugin.forward_context(): + with self.precision_plugin.forward_context(): return self.training_type_plugin.validation_step(*step_kwargs.values()) def test_step(self, step_kwargs: Dict[str, Union[Any, int]]) -> Optional[STEP_OUTPUT]: @@ -239,7 +239,7 @@ def test_step(self, step_kwargs: Dict[str, Union[Any, int]]) -> Optional[STEP_OU See :meth:`~pytorch_lightning.core.lightning.LightningModule.test_step` for more details """ - with self.precision_plugin.forward_context(), self.training_type_plugin.forward_context(): + with self.precision_plugin.forward_context(): return self.training_type_plugin.test_step(*step_kwargs.values()) def predict_step(self, step_kwargs: Dict[str, Union[Any, int]]) -> STEP_OUTPUT: @@ -247,7 +247,7 @@ def predict_step(self, step_kwargs: Dict[str, Union[Any, int]]) -> STEP_OUTPUT: See :meth:`~pytorch_lightning.core.lightning.LightningModule.predict_step` for more details """ - with self.precision_plugin.forward_context(), self.training_type_plugin.forward_context(): + with self.precision_plugin.forward_context(): return self.training_type_plugin.predict_step(*step_kwargs.values()) def training_step_end(self, output: STEP_OUTPUT) -> STEP_OUTPUT: @@ -709,8 +709,3 @@ def on_train_batch_start(self, batch: Any, batch_idx: int, dataloader_idx: int = "`on_train_batch_start` logic is implemented directly in the `TrainingTypePlugin` implementations." ) return self.training_type_plugin.on_train_batch_start(batch, batch_idx) - - @contextlib.contextmanager - def forward_context(self): - with self.precision_plugin.forward_context(), self.training_type_plugin.forward_context(): - yield diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index aa8ea606cfc5e..c613d6744ab89 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -239,7 +239,7 @@ def cast(self) -> Generator[None, None, None]: Use this only if the `forward` method of your model does not cover all operations you wish to run with the chosen precision setting. """ - with self._accelerator.forward_context(): + with self._precision_plugin.forward_context(): yield def to_device(self, obj: Union[nn.Module, Tensor, Any]) -> Union[nn.Module, Tensor, Any]: diff --git a/pytorch_lightning/plugins/training_type/training_type_plugin.py b/pytorch_lightning/plugins/training_type/training_type_plugin.py index 64a1ed6b32c2e..e0bc057cf4b41 100644 --- a/pytorch_lightning/plugins/training_type/training_type_plugin.py +++ b/pytorch_lightning/plugins/training_type/training_type_plugin.py @@ -300,11 +300,6 @@ def remove_checkpoint(self, filepath: _PATH) -> None: if self.should_rank_save_checkpoint: return self.checkpoint_io.remove_checkpoint(filepath) - @contextlib.contextmanager - def forward_context(self) -> Generator: - """A contextmanager for managing model forward/training_step/evaluation_step/predict_step.""" - yield - @contextlib.contextmanager def model_sharded_context(self) -> Generator: """Provide hook to create modules in a distributed aware context. This is useful for when we'd like to From e03499aea8f8fe6bb7d7513ac47088dabc13377e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 18 Oct 2021 13:20:51 +0200 Subject: [PATCH 106/331] refactor forward context --- pytorch_lightning/plugins/precision/double.py | 33 +------------------ .../plugins/precision/native_amp.py | 24 -------------- .../plugins/precision/precision_plugin.py | 22 ++++++++----- 3 files changed, 14 insertions(+), 65 deletions(-) diff --git a/pytorch_lightning/plugins/precision/double.py b/pytorch_lightning/plugins/precision/double.py index 179daf9e91db8..2b104f321ad38 100644 --- a/pytorch_lightning/plugins/precision/double.py +++ b/pytorch_lightning/plugins/precision/double.py @@ -91,38 +91,7 @@ def connect( return super().connect(model, optimizers, lr_schedulers) - @contextmanager - def train_step_context(self) -> Generator[None, None, None]: - """A context manager to change the default tensor type. - - See: :meth:`torch.set_default_tensor_type` - """ - torch.set_default_tensor_type(torch.DoubleTensor) - yield - torch.set_default_tensor_type(torch.FloatTensor) - - @contextmanager - def val_step_context(self) -> Generator[None, None, None]: - """A context manager to change the default tensor type. - - See: :meth:`torch.set_default_tensor_type` - """ - torch.set_default_tensor_type(torch.DoubleTensor) - yield - torch.set_default_tensor_type(torch.FloatTensor) - - @contextmanager - def test_step_context(self) -> Generator[None, None, None]: - """A context manager to change the default tensor type. - - See: :meth:`torch.set_default_tensor_type` - """ - torch.set_default_tensor_type(torch.DoubleTensor) - yield - torch.set_default_tensor_type(torch.FloatTensor) - - @contextmanager - def predict_step_context(self) -> Generator[None, None, None]: + def forward_context(self) -> Generator[None, None, None]: """A context manager to change the default tensor type. See: :meth:`torch.set_default_tensor_type` diff --git a/pytorch_lightning/plugins/precision/native_amp.py b/pytorch_lightning/plugins/precision/native_amp.py index 97aa134f5f99b..92c6d32d8ce58 100644 --- a/pytorch_lightning/plugins/precision/native_amp.py +++ b/pytorch_lightning/plugins/precision/native_amp.py @@ -123,30 +123,6 @@ def forward_context(self) -> Generator[None, None, None]: with self.autocast_context_manager(): yield - @contextmanager - def train_step_context(self) -> Generator[None, None, None]: - """Enable autocast context.""" - with self.autocast_context_manager(): - yield - - @contextmanager - def val_step_context(self) -> Generator[None, None, None]: - """Enable autocast context.""" - with self.autocast_context_manager(): - yield - - @contextmanager - def test_step_context(self) -> Generator[None, None, None]: - """Enable autocast context.""" - with self.autocast_context_manager(): - yield - - @contextmanager - def predict_step_context(self) -> Generator[None, None, None]: - """Enable autocast context.""" - with self.autocast_context_manager(): - yield - def on_load_checkpoint(self, checkpoint: Dict[str, Any]) -> None: if "native_amp_scaling_state" in checkpoint and not self.is_bfloat16: self.scaler.load_state_dict(checkpoint["native_amp_scaling_state"]) diff --git a/pytorch_lightning/plugins/precision/precision_plugin.py b/pytorch_lightning/plugins/precision/precision_plugin.py index fd7ee03f7154d..6192d51c86f27 100644 --- a/pytorch_lightning/plugins/precision/precision_plugin.py +++ b/pytorch_lightning/plugins/precision/precision_plugin.py @@ -157,26 +157,30 @@ def post_dispatch(self) -> None: """Hook to do something after the training/evaluation/prediction finishes.""" @contextlib.contextmanager - def forward_context(self) -> Generator: + def forward_context(self) -> Generator[None, None, None]: """A contextmanager for managing model forward/training_step/evaluation_step/predict_step.""" yield @contextlib.contextmanager - def train_step_context(self) -> Generator: + def train_step_context(self) -> Generator[None, None, None]: """A contextmanager for the training step.""" - yield + with self.forward_context(): + yield @contextlib.contextmanager - def val_step_context(self) -> Generator: + def val_step_context(self) -> Generator[None, None, None]: """A contextmanager for the validation step.""" - yield + with self.forward_context(): + yield @contextlib.contextmanager - def test_step_context(self) -> Generator: + def test_step_context(self) -> Generator[None, None, None]: """A contextmanager for the test step.""" - yield + with self.forward_context(): + yield @contextlib.contextmanager - def predict_step_context(self) -> Generator: + def predict_step_context(self) -> Generator[None, None, None]: """A contextmanager for the predict step.""" - yield + with self.forward_context(): + yield From 2daa95d3d43c8c1d20311912a3a91368d61ca818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 18 Oct 2021 13:22:26 +0200 Subject: [PATCH 107/331] revert accelerator changes of context managers --- pytorch_lightning/accelerators/accelerator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/accelerators/accelerator.py b/pytorch_lightning/accelerators/accelerator.py index 2771a3813bce0..96f1aea73dd14 100644 --- a/pytorch_lightning/accelerators/accelerator.py +++ b/pytorch_lightning/accelerators/accelerator.py @@ -211,7 +211,7 @@ def training_step(self, step_kwargs: Dict[str, Union[Any, int]]) -> STEP_OUTPUT: See :meth:`~pytorch_lightning.core.lightning.LightningModule.training_step` for more details """ - with self.precision_plugin.forward_context(): + with self.precision_plugin.train_step_context(): return self.training_type_plugin.training_step(*step_kwargs.values()) def post_training_step(self) -> None: @@ -231,7 +231,7 @@ def validation_step(self, step_kwargs: Dict[str, Union[Any, int]]) -> Optional[S See :meth:`~pytorch_lightning.core.lightning.LightningModule.validation_step` for more details """ - with self.precision_plugin.forward_context(): + with self.precision_plugin.val_step_context(): return self.training_type_plugin.validation_step(*step_kwargs.values()) def test_step(self, step_kwargs: Dict[str, Union[Any, int]]) -> Optional[STEP_OUTPUT]: @@ -239,7 +239,7 @@ def test_step(self, step_kwargs: Dict[str, Union[Any, int]]) -> Optional[STEP_OU See :meth:`~pytorch_lightning.core.lightning.LightningModule.test_step` for more details """ - with self.precision_plugin.forward_context(): + with self.precision_plugin.test_step_context(): return self.training_type_plugin.test_step(*step_kwargs.values()) def predict_step(self, step_kwargs: Dict[str, Union[Any, int]]) -> STEP_OUTPUT: @@ -247,7 +247,7 @@ def predict_step(self, step_kwargs: Dict[str, Union[Any, int]]) -> STEP_OUTPUT: See :meth:`~pytorch_lightning.core.lightning.LightningModule.predict_step` for more details """ - with self.precision_plugin.forward_context(): + with self.precision_plugin.predict_step_context(): return self.training_type_plugin.predict_step(*step_kwargs.values()) def training_step_end(self, output: STEP_OUTPUT) -> STEP_OUTPUT: From 8b5eb80ae64f0ef8bc74b6666d116c338c063535 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 18 Oct 2021 13:24:25 +0200 Subject: [PATCH 108/331] fix model wrapper call to forward context --- pytorch_lightning/lite/wrappers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 7a332952ee15e..130b495604c4a 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -77,7 +77,7 @@ def module(self) -> nn.Module: return self._module def forward(self, *args: Any, **kwargs: Any) -> Any: - with self._accelerator.forward_context(): + with self._accelerator.precision_plugin.forward_context(): output = self.module.forward(*args, **kwargs) output = apply_to_collection(output, function=lambda t: t.to(torch.get_default_dtype()), dtype=Tensor) From 0284098ed4ccc6433e6dba53856d968597ba3efa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 18 Oct 2021 14:00:32 +0200 Subject: [PATCH 109/331] Update pytorch_lightning/lite/wrappers.py Co-authored-by: Justus Schock <12886177+justusschock@users.noreply.github.com> --- pytorch_lightning/lite/wrappers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 130b495604c4a..ad2804469428b 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -89,8 +89,7 @@ def __init__(self, device: Optional[torch.device] = None, **dl_kwargs: Any) -> N super().__init__(**dl_kwargs) self._device = device - # TODO: how to type this *angry face" - def __iter__(self): # type: ignore + def __iter__(self) -> Union[Iterator[Any], Generator[Any, None, None]] iterator = super().__iter__() if self._device is None: return iterator From 09eb2066995ac416277bdaedd52de1cf9382f433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 18 Oct 2021 14:02:02 +0200 Subject: [PATCH 110/331] add missing imports for type --- pytorch_lightning/lite/wrappers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index ad2804469428b..4d50f9d19c912 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Callable, Optional +from typing import Any, Callable, Optional, Union, Iterator, Generator import torch from torch import nn as nn @@ -89,7 +89,7 @@ def __init__(self, device: Optional[torch.device] = None, **dl_kwargs: Any) -> N super().__init__(**dl_kwargs) self._device = device - def __iter__(self) -> Union[Iterator[Any], Generator[Any, None, None]] + def __iter__(self) -> Union[Iterator[Any], Generator[Any, None, None]]: iterator = super().__iter__() if self._device is None: return iterator From 1b6db9bcd96e89387b03871a58410d1f3e186d8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 18 Oct 2021 14:03:07 +0200 Subject: [PATCH 111/331] add override ignore type for __iter__ --- pytorch_lightning/lite/wrappers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 4d50f9d19c912..0f1233f1b5f92 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -89,7 +89,7 @@ def __init__(self, device: Optional[torch.device] = None, **dl_kwargs: Any) -> N super().__init__(**dl_kwargs) self._device = device - def __iter__(self) -> Union[Iterator[Any], Generator[Any, None, None]]: + def __iter__(self) -> Union[Iterator[Any], Generator[Any, None, None]]: # type: ignore[override] iterator = super().__iter__() if self._device is None: return iterator From 53c1748a1a0a0a7ae9d1b1ad6f9a6ef7ebff3da4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Oct 2021 12:04:29 +0000 Subject: [PATCH 112/331] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytorch_lightning/lite/wrappers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 0f1233f1b5f92..94cd8ecb89e5d 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Callable, Optional, Union, Iterator, Generator +from typing import Any, Callable, Generator, Iterator, Optional, Union import torch from torch import nn as nn From ae12a4d42e0e303e54f50b159fd5f77a938937b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 18 Oct 2021 14:32:35 +0200 Subject: [PATCH 113/331] update setup logic for ddp --- pytorch_lightning/lite/lite.py | 2 +- pytorch_lightning/plugins/training_type/ddp.py | 11 ++--------- pytorch_lightning/plugins/training_type/ddp_spawn.py | 8 +++++--- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index c613d6744ab89..0db786b4a93db 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -156,11 +156,11 @@ def setup( """ # wrap all objects passed in and return them in the same order optimizers = [optimizers] if isinstance(optimizers, Optimizer) else optimizers - model, optimizers = self._setup_model_and_optimizers(model, optimizers) if move_to_device: model = self.to_device(model) + model, optimizers = self._setup_model_and_optimizers(model, optimizers) optimizers = optimizers[0] if len(optimizers) == 1 else optimizers return model, optimizers diff --git a/pytorch_lightning/plugins/training_type/ddp.py b/pytorch_lightning/plugins/training_type/ddp.py index 7fc53b2b05b3d..2bcb032e3adcf 100644 --- a/pytorch_lightning/plugins/training_type/ddp.py +++ b/pytorch_lightning/plugins/training_type/ddp.py @@ -184,12 +184,7 @@ def setup_environment(self) -> None: self.setup_distributed() def setup_model(self, model: Module) -> Module: - model = DistributedDataParallel( - module=model.to(self.root_device), - device_ids=self.determine_ddp_device_ids(), - **self._ddp_kwargs, - ) - return model + return DistributedDataParallel(module=model, device_ids=self.determine_ddp_device_ids(), **self._ddp_kwargs) def _call_children_scripts(self): # bookkeeping of spawned processes @@ -365,9 +360,7 @@ def _reinit_optimizers_with_post_localSGD(self, warmup_steps: int): def configure_ddp(self) -> None: self.pre_configure_ddp() - self._model = DistributedDataParallel( - LightningDistributedModule(self.model), device_ids=self.determine_ddp_device_ids(), **self._ddp_kwargs - ) + self._model = self.setup_model(LightningDistributedModule(self.model)) self._register_ddp_hooks() def determine_ddp_device_ids(self): diff --git a/pytorch_lightning/plugins/training_type/ddp_spawn.py b/pytorch_lightning/plugins/training_type/ddp_spawn.py index 07fa4e455df26..7e5433cb4beba 100644 --- a/pytorch_lightning/plugins/training_type/ddp_spawn.py +++ b/pytorch_lightning/plugins/training_type/ddp_spawn.py @@ -22,6 +22,7 @@ import torch import torch.distributed import torch.multiprocessing as mp +from torch.nn import Module from torch.nn.parallel.distributed import DistributedDataParallel import pytorch_lightning as pl @@ -148,6 +149,9 @@ def setup(self) -> None: smp = mp.get_context("spawn") self.mp_queue = smp.SimpleQueue() + def setup_model(self, model: Module) -> Module: + return DistributedDataParallel(module=model, device_ids=self.determine_ddp_device_ids(), **self._ddp_kwargs) + def set_world_ranks(self, process_idx: int = 0) -> None: self._local_rank = process_idx if self.cluster_environment is None: @@ -259,9 +263,7 @@ def _register_ddp_hooks(self) -> None: def configure_ddp(self) -> None: self.pre_configure_ddp() - self._model = DistributedDataParallel( - LightningDistributedModule(self.model), device_ids=self.determine_ddp_device_ids(), **self._ddp_kwargs - ) + self._model = self.setup_model(LightningDistributedModule(self.model)) self._register_ddp_hooks() def determine_ddp_device_ids(self): From 9e4a9d0c758a829dc892fcafae77ebc9ad86ee3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 18 Oct 2021 14:50:24 +0200 Subject: [PATCH 114/331] remove proxy changes from connector --- pytorch_lightning/lite/lite.py | 2 +- pytorch_lightning/trainer/connectors/accelerator_connector.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 0db786b4a93db..8029ed9a25dce 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -93,7 +93,7 @@ def __init__( amp_level=None, plugins=plugins, ) - self._accelerator = self._accelerator_connector.select_accelerator() + self._accelerator = self._accelerator_connector.accelerator self._strategy = self._accelerator.training_type_plugin self._precision_plugin = self._accelerator.precision_plugin diff --git a/pytorch_lightning/trainer/connectors/accelerator_connector.py b/pytorch_lightning/trainer/connectors/accelerator_connector.py index 7ebc183e624e8..53f95ae4c8a14 100644 --- a/pytorch_lightning/trainer/connectors/accelerator_connector.py +++ b/pytorch_lightning/trainer/connectors/accelerator_connector.py @@ -763,8 +763,8 @@ def select_accelerator(self) -> Accelerator: # that we first select training_type_plugin, then precision_plugin accelerator = acc_cls(training_type_plugin=self.training_type_plugin, precision_plugin=self.precision_plugin) # transfer ownership of the plugins to the accelerator - self._training_type_plugin = self.training_type_plugin - self._precision_plugin = self.precision_plugin + self._training_type_plugin = proxy(self.training_type_plugin) + self._precision_plugin = proxy(self.precision_plugin) return accelerator From 0f4b790d3637eb15717d9524d954dffa2737ad20 Mon Sep 17 00:00:00 2001 From: SeanNaren Date: Mon, 18 Oct 2021 14:57:20 +0100 Subject: [PATCH 115/331] Hack to allow deepspeed to run fp16 --- pytorch_lightning/lite/lite.py | 36 +++++++++++++------ .../plugins/training_type/deepspeed.py | 15 ++++---- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 76d5a6607c8e2..6ede74f817157 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -12,26 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -from abc import abstractmethod, ABC +from abc import ABC, abstractmethod from collections import Callable from contextlib import contextmanager from functools import partial from pathlib import Path -from typing import Any, Optional, Sequence, Union, List, Dict, Tuple, Generator +from typing import Any, Dict, Generator, List, Optional, Sequence, Tuple, Union import torch import torch.nn as nn from torch import Tensor from torch.optim import Optimizer -from torch.utils.data import DataLoader, DistributedSampler, SequentialSampler, RandomSampler, Sampler +from torch.utils.data import DataLoader, DistributedSampler, RandomSampler, SequentialSampler from pytorch_lightning import Trainer from pytorch_lightning.accelerators import Accelerator, TPUAccelerator -from pytorch_lightning.lite.wrappers import _LiteOptimizer, _LiteModule, _LiteDataLoader -from pytorch_lightning.plugins import PLUGIN_INPUT, DDPSpawnPlugin, TrainingTypePlugin, DeepSpeedPlugin +from pytorch_lightning.lite.wrappers import _LiteDataLoader, _LiteModule, _LiteOptimizer +from pytorch_lightning.plugins import DDPSpawnPlugin, DeepSpeedPlugin, PLUGIN_INPUT, TrainingTypePlugin from pytorch_lightning.trainer.connectors.accelerator_connector import AcceleratorConnector from pytorch_lightning.trainer.data_loading import TrainerDataLoadingMixin -from pytorch_lightning.utilities import move_data_to_device, DistributedType, DeviceType +from pytorch_lightning.utilities import DeviceType, DistributedType, move_data_to_device from pytorch_lightning.utilities.data import has_iterable_dataset from pytorch_lightning.utilities.exceptions import MisconfigurationException @@ -102,7 +102,10 @@ def __init__( @property def device(self) -> torch.device: - """The current device this process runs on. Use this to create tensors directly on the device if needed.""" + """The current device this process runs on. + + Use this to create tensors directly on the device if needed. + """ return self._accelerator.root_device @property @@ -233,8 +236,8 @@ def backward(self, tensor: Tensor, *args: Any, **kwargs: Any) -> None: def cast(self) -> Generator[None, None, None]: """A context manager to automatically convert operations for the chosen precision. - Use this only if the `forward` method of your model does not cover all operations you wish to run with - the chosen precision setting. + Use this only if the `forward` method of your model does not cover all operations you wish to run with the + chosen precision setting. """ with self._accelerator.forward_context(): yield @@ -255,8 +258,10 @@ def to_device(self, obj: Union[nn.Module, Tensor, Any]) -> Union[nn.Module, Tens return move_data_to_device(obj, device=self.device) def print(self, *args: Any, **kwargs: Any) -> None: - """Print something only on the first process. Arguments passed to this method are forwarded to the - Python built-in :func:`print` function.""" + """Print something only on the first process. + + Arguments passed to this method are forwarded to the Python built-in :func:`print` function. + """ if self.local_rank == 0: print(*args, **kwargs) @@ -291,6 +296,9 @@ def _run_wrapper(self, run_method: Callable) -> Callable: return partial(self._run_impl, run_method) def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> None: + if isinstance(self._strategy, DeepSpeedPlugin): + # todo: this is a hack as deepspeed currently relies on the precision plugin + self._set_deepspeed_precision_variables() self._strategy.setup_environment() if isinstance(self._strategy, DDPSpawnPlugin): self._strategy.spawn(run_method, *args, **kwargs) @@ -298,6 +306,12 @@ def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> None: run_method(*args, **kwargs) # TODO: any teardown needed here? + def _set_deepspeed_precision_variables(self): + amp_type = self._accelerator_connector.amp_type + amp_level = self._accelerator_connector.amp_level + precision = self._accelerator_connector.precision + self._strategy.amp_level, self._strategy.amp_type, self._strategy._precision = amp_level, amp_type, precision + def _setup_model_and_optimizers( self, model: nn.Module, diff --git a/pytorch_lightning/plugins/training_type/deepspeed.py b/pytorch_lightning/plugins/training_type/deepspeed.py index 07ea169f7d5ff..ca2e279ddbba3 100644 --- a/pytorch_lightning/plugins/training_type/deepspeed.py +++ b/pytorch_lightning/plugins/training_type/deepspeed.py @@ -325,6 +325,10 @@ def __init__( self.hysteresis = hysteresis self.min_loss_scale = min_loss_scale + self._precision = None + self.amp_level = None + self.amp_type = None + def _load_config(self, config): if config is None and self.DEEPSPEED_ENV_VAR in os.environ: rank_zero_info(f"Loading DeepSpeed config from set {self.DEEPSPEED_ENV_VAR} environment variable") @@ -516,7 +520,7 @@ def model_sharded_context(self) -> Generator[None, None, None]: @property def precision(self) -> Union[str, int]: - return self.lightning_module.trainer.precision + return self._precision or self.lightning_module.trainer.precision def _set_deepspeed_activation_checkpointing(self): if self.config.get("activation_checkpointing"): @@ -633,11 +637,10 @@ def _auto_select_batch_size(self): return batch_size def _format_precision_config(self): - # TODO: support precision - return - amp_type = self.lightning_module.trainer.accelerator_connector.amp_type - amp_level = self.lightning_module.trainer.accelerator_connector.amp_level - precision = self.lightning_module.trainer.accelerator_connector.precision + amp_type = self.amp_type or self.lightning_module.trainer.accelerator_connector.amp_type + precision = self.precision or self.lightning_module.trainer.accelerator_connector.precision + if amp_type == AMPType.APEX: + amp_level = self.amp_level or self.lightning_module.trainer.accelerator_connector.amp_level if precision in (16, "mixed"): if "fp16" not in self.config and amp_type == AMPType.NATIVE: # FP16 is a DeepSpeed standalone AMP implementation From 520228f2eed9772c04f8be40121caff9e2f671c0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Oct 2021 14:19:19 +0000 Subject: [PATCH 116/331] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytorch_lightning/lite/lite.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 615e0721900cb..3d0d43ae0ee89 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -25,7 +25,6 @@ from torch.optim import Optimizer from torch.utils.data import DataLoader, DistributedSampler, RandomSampler, SequentialSampler - from pytorch_lightning import Trainer from pytorch_lightning.accelerators import Accelerator, TPUAccelerator from pytorch_lightning.lite.wrappers import _LiteDataLoader, _LiteModule, _LiteOptimizer From 7f8aeeabcb515217d3daaee28666d67d636e0756 Mon Sep 17 00:00:00 2001 From: Sean Naren Date: Mon, 18 Oct 2021 16:48:58 +0100 Subject: [PATCH 117/331] Support sharded ddp precision check (#3) --- pytorch_lightning/lite/lite.py | 21 ++++++++++++++----- .../plugins/training_type/sharded.py | 10 ++++++--- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 615e0721900cb..cb08e0ea6fbc2 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -29,7 +29,13 @@ from pytorch_lightning import Trainer from pytorch_lightning.accelerators import Accelerator, TPUAccelerator from pytorch_lightning.lite.wrappers import _LiteDataLoader, _LiteModule, _LiteOptimizer -from pytorch_lightning.plugins import DDPSpawnPlugin, DeepSpeedPlugin, PLUGIN_INPUT, TrainingTypePlugin +from pytorch_lightning.plugins import ( + DDPShardedPlugin, + DDPSpawnPlugin, + DeepSpeedPlugin, + PLUGIN_INPUT, + TrainingTypePlugin, +) from pytorch_lightning.trainer.connectors.accelerator_connector import AcceleratorConnector from pytorch_lightning.trainer.data_loading import TrainerDataLoadingMixin from pytorch_lightning.utilities import DeviceType, DistributedType, move_data_to_device @@ -297,9 +303,7 @@ def _run_wrapper(self, run_method: Callable) -> Callable: return partial(self._run_impl, run_method) def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> None: - if isinstance(self._strategy, DeepSpeedPlugin): - # todo: this is a hack as deepspeed currently relies on the precision plugin - self._set_deepspeed_precision_variables() + self._set_plugin_specific_precision_variables() self._strategy.setup_environment() if isinstance(self._strategy, DDPSpawnPlugin): self._strategy.spawn(run_method, *args, **kwargs) @@ -307,7 +311,14 @@ def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> None: run_method(*args, **kwargs) # TODO: any teardown needed here? - def _set_deepspeed_precision_variables(self): + def _set_plugin_specific_precision_variables(self) -> None: + # todo: these are hacks as plugins rely on access to the precision plugin + if isinstance(self._strategy, DeepSpeedPlugin): + self._set_deepspeed_precision_variables() + if isinstance(self._strategy, DDPShardedPlugin): + self._strategy._precision = self._accelerator_connector.precision + + def _set_deepspeed_precision_variables(self) -> None: amp_type = self._accelerator_connector.amp_type amp_level = self._accelerator_connector.amp_level precision = self._accelerator_connector.precision diff --git a/pytorch_lightning/plugins/training_type/sharded.py b/pytorch_lightning/plugins/training_type/sharded.py index 2b194fe462cf4..099f7a24bfd0a 100644 --- a/pytorch_lightning/plugins/training_type/sharded.py +++ b/pytorch_lightning/plugins/training_type/sharded.py @@ -35,7 +35,11 @@ class DDPShardedPlugin(DDPPlugin): """Optimizer and gradient sharded training provided by FairScale.""" - _REDUCE_BUFFER_SIZE_DEFAULT = 2 ** 23 # 8M + _REDUCE_BUFFER_SIZE_DEFAULT: int = 2 ** 23 # 8M + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._precision = None def setup_models_and_optimizers( self, models: List[Module], optimizers: List[Optimizer] @@ -71,9 +75,9 @@ def _reinit_optimizers_with_oss(self, optimizers: List[Union[Optimizer, Lightnin optim_class = type(optimizer) zero_optimizer = OSS(params=optimizer.param_groups, optim=optim_class, **optimizer.defaults) if _FAIRSCALE_OSS_FP16_BROADCAST_AVAILABLE: - precision = ( + precision = self._precision or ( 32 if self.lightning_module is None else self.lightning_module.trainer.precision - ) # TODO: how to handle this?! + ) is_fp16 = precision in ("mixed", 16) # For multi-node training, compressing the model shards in fp16 before broadcasting # improves performance. When using PyTorch AMP, it will not degrade From 4642459809f9bf15f7c72e32ce6ab5a4f9110b0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 18 Oct 2021 18:07:22 +0200 Subject: [PATCH 118/331] call run_backward on precision plugin directly --- pytorch_lightning/accelerators/accelerator.py | 4 ---- pytorch_lightning/lite/lite.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/pytorch_lightning/accelerators/accelerator.py b/pytorch_lightning/accelerators/accelerator.py index 96f1aea73dd14..7e86f9f700986 100644 --- a/pytorch_lightning/accelerators/accelerator.py +++ b/pytorch_lightning/accelerators/accelerator.py @@ -314,10 +314,6 @@ def backward(self, closure_loss: Tensor, *args: Any, **kwargs: Any) -> Tensor: return closure_loss - def run_backward(self, tensor: Tensor, model, *args, **kwargs) -> None: - """Lightning-independent backward logic.""" - self.precision_plugin.run_backward(tensor, model, *args, **kwargs) - def optimizer_step( self, optimizer: Optimizer, diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index db00cebe8ce1f..f57c788e66b06 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -236,7 +236,7 @@ def backward(self, tensor: Tensor, *args: Any, **kwargs: Any) -> None: *args: Optional positional arguments passed to the underlying backward function. **kwargs: Optional named keyword arguments passed to the underlying backward function. """ - self._accelerator.run_backward(tensor, self._strategy.model, *args, **kwargs) + self._precision_plugin.run_backward(tensor, self._strategy.model, *args, **kwargs) @contextmanager def cast(self) -> Generator[None, None, None]: From 7a5da9c52bd2c8df20e3af2ae0cf92199803d07a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 19 Oct 2021 02:13:52 +0200 Subject: [PATCH 119/331] address todo for deepspeed --- .../plugins/training_type/deepspeed.py | 29 +++++-------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/pytorch_lightning/plugins/training_type/deepspeed.py b/pytorch_lightning/plugins/training_type/deepspeed.py index ca2e279ddbba3..7ce4a14398c6a 100644 --- a/pytorch_lightning/plugins/training_type/deepspeed.py +++ b/pytorch_lightning/plugins/training_type/deepspeed.py @@ -24,6 +24,7 @@ import torch from torch.nn import Module from torch.optim import Optimizer +from torch.optim.lr_scheduler import _LRScheduler import pytorch_lightning as pl from pytorch_lightning.overrides.base import _LightningModuleWrapperBase @@ -383,7 +384,6 @@ def pre_dispatch(self): self.init_deepspeed() self.barrier() - # TODO: avoid code duplication by letting the plugin reuse this method def setup_models_and_optimizers( self, models: List[Module], optimizers: List[Optimizer] ) -> Tuple[List[Module], List[Optimizer]]: @@ -393,24 +393,22 @@ def setup_models_and_optimizers( f" Got {len(models)} models and {len(optimizers)} optimizers instead." ) - # TODO: is this the correct place to set this? self.config["train_micro_batch_size_per_gpu"] = 1 - self._model, optimizer = self._setup_model_and_optimizer(models[0], optimizers[0]) - - # TODO: do we need to call it here? - # self._set_deepspeed_activation_checkpointing() + self._set_deepspeed_activation_checkpointing() return [self._model], [optimizer] - def _setup_model_and_optimizer(self, model: Module, optimizer: Optimizer): - # TODO: shouldn't this be optimizer.parameters? + def _setup_model_and_optimizer( + self, model: Module, optimizer: Optimizer, lr_scheduler: Optional[_LRScheduler] = None + ): model_parameters = filter(lambda p: p.requires_grad, model.parameters()) deepspeed_engine, deepspeed_optimizer, _, _ = deepspeed.initialize( args=argparse.Namespace(device_rank=self.root_device.index), config=self.config, model=model, - model_parameters=model_parameters, # TODO: is the type correct here? + model_parameters=model_parameters, # type: ignore optimizer=optimizer, + lr_scheduler=lr_scheduler, dist_init_required=False, ) return deepspeed_engine, deepspeed_optimizer @@ -479,18 +477,7 @@ def _initialize_deepspeed_train(self, model): optimizer, lr_scheduler, _ = self._init_optimizers() scheduler = lr_scheduler["scheduler"] - - model_parameters = filter(lambda p: p.requires_grad, self.model.parameters()) - model, deepspeed_optimizer, _, deepspeed_scheduler = deepspeed.initialize( - args=argparse.Namespace(device_rank=self.root_device.index), - config=self.config, - model=model, - model_parameters=model_parameters, - optimizer=optimizer, - lr_scheduler=scheduler, - dist_init_required=False, - ) - + model, deepspeed_optimizer = self._setup_model_and_optimizer(model, optimizer, scheduler) self._set_deepspeed_activation_checkpointing() # although we set these here, deepspeed manages the specific optimizer logic From 8ddb777f5e1a1d44df3fec37bb80daaad521387d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 19 Oct 2021 02:17:38 +0200 Subject: [PATCH 120/331] move scheduler --- pl_examples/lite_examples/simple/mnist_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/lite_examples/simple/mnist_example.py b/pl_examples/lite_examples/simple/mnist_example.py index 1d9cb715c4137..abb4765b6a603 100644 --- a/pl_examples/lite_examples/simple/mnist_example.py +++ b/pl_examples/lite_examples/simple/mnist_example.py @@ -62,13 +62,13 @@ def run(self, args): model = Net() optimizer = optim.Adadelta(model.parameters(), lr=args.lr) + scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) train_loader, test_loader = self.setup_dataloaders(train_loader, test_loader) assert isinstance(train_loader.sampler, DistributedSampler) assert isinstance(test_loader.sampler, DistributedSampler) model, optimizer = self.setup(model, optimizer) - scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) for epoch in range(1, args.epochs + 1): self.train(args, model, train_loader, optimizer, epoch) self.test(model, test_loader) From c2b4b74d15de163f8f3074f586c4694de29a25ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 19 Oct 2021 02:30:42 +0200 Subject: [PATCH 121/331] convert --- pytorch_lightning/lite/wrappers.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 94cd8ecb89e5d..c6e3c6a113880 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -77,6 +77,17 @@ def module(self) -> nn.Module: return self._module def forward(self, *args: Any, **kwargs: Any) -> Any: + precision = self._accelerator.precision_plugin.precision + precision_to_type = { + "mixed": torch.half, + 16: torch.half, + 32: torch.float, + 64: torch.double, + } + # TODO (@awaelchli): let the precision plugin handle the conversion + to_type = precision_to_type[precision] + args, kwargs = apply_to_collection([args, kwargs], function=lambda t: t.to(to_type), dtype=Tensor) + with self._accelerator.precision_plugin.forward_context(): output = self.module.forward(*args, **kwargs) From 0d430c51fc168da4863d8f57af977197beb41232 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 19 Oct 2021 03:12:29 +0200 Subject: [PATCH 122/331] fix precision + dataloader wrapping for DP --- pytorch_lightning/lite/lite.py | 20 +++++++++---------- pytorch_lightning/lite/wrappers.py | 11 ++++++++++ pytorch_lightning/plugins/training_type/dp.py | 1 - 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index f57c788e66b06..6a78ba46894bc 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -211,19 +211,19 @@ def _setup_dataloader( Returns: The wrapped dataloader. """ - if not replace_sampler or not ( + sampler = dataloader.sampler + if replace_sampler and ( self._requires_distributed_sampler(dataloader) or isinstance(self._accelerator, TPUAccelerator) ): - return dataloader - if not isinstance(dataloader.sampler, (SequentialSampler, RandomSampler)): - raise MisconfigurationException( - "You seem to have configured a sampler in your DataLoader. This will be replaced " - " by `DistributedSampler` since `replace_sampler_ddp` is True and you are using" - " distributed training. Either remove the sampler from your DataLoader or set" - " `replace_sampler=False` if you want to use your custom sampler." - ) + if not isinstance(dataloader.sampler, (SequentialSampler, RandomSampler)): + raise MisconfigurationException( + "You seem to have configured a sampler in your DataLoader. This will be replaced " + " by `DistributedSampler` since `replace_sampler_ddp` is True and you are using" + " distributed training. Either remove the sampler from your DataLoader or set" + " `replace_sampler=False` if you want to use your custom sampler." + ) + sampler = self._get_distributed_sampler(dataloader, **self._strategy.distributed_sampler_kwargs) - sampler = self._get_distributed_sampler(dataloader, **self._strategy.distributed_sampler_kwargs) kwargs = TrainerDataLoadingMixin._get_dataloader_init_kwargs(dataloader, sampler) device = self.device if move_to_device else None return _LiteDataLoader(device=device, **kwargs) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 94cd8ecb89e5d..c6e3c6a113880 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -77,6 +77,17 @@ def module(self) -> nn.Module: return self._module def forward(self, *args: Any, **kwargs: Any) -> Any: + precision = self._accelerator.precision_plugin.precision + precision_to_type = { + "mixed": torch.half, + 16: torch.half, + 32: torch.float, + 64: torch.double, + } + # TODO (@awaelchli): let the precision plugin handle the conversion + to_type = precision_to_type[precision] + args, kwargs = apply_to_collection([args, kwargs], function=lambda t: t.to(to_type), dtype=Tensor) + with self._accelerator.precision_plugin.forward_context(): output = self.module.forward(*args, **kwargs) diff --git a/pytorch_lightning/plugins/training_type/dp.py b/pytorch_lightning/plugins/training_type/dp.py index a5a346f82698c..81b4e94d91f6d 100644 --- a/pytorch_lightning/plugins/training_type/dp.py +++ b/pytorch_lightning/plugins/training_type/dp.py @@ -15,7 +15,6 @@ import torch from torch.nn import DataParallel, Module -from torch.optim import Optimizer from pytorch_lightning.overrides.data_parallel import LightningParallelModule from pytorch_lightning.plugins.io.checkpoint_plugin import CheckpointIO From 2a21ad99fdf4df1e0d071fd4bc48162470be04e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 19 Oct 2021 03:19:48 +0200 Subject: [PATCH 123/331] remove unused import --- pytorch_lightning/utilities/distributed.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pytorch_lightning/utilities/distributed.py b/pytorch_lightning/utilities/distributed.py index 88d9be04e211e..76dafbaadeaeb 100644 --- a/pytorch_lightning/utilities/distributed.py +++ b/pytorch_lightning/utilities/distributed.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import inspect import logging import os from functools import wraps From eeff8431c4e3f850f22e1e3a1732b620c1b810cb Mon Sep 17 00:00:00 2001 From: Kaushik B <45285388+kaushikb11@users.noreply.github.com> Date: Tue, 19 Oct 2021 12:12:34 +0530 Subject: [PATCH 124/331] Update pl_examples/lite_examples/simple/mnist_example.py --- pl_examples/lite_examples/simple/mnist_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/lite_examples/simple/mnist_example.py b/pl_examples/lite_examples/simple/mnist_example.py index abb4765b6a603..2bd2ccaedb0b0 100644 --- a/pl_examples/lite_examples/simple/mnist_example.py +++ b/pl_examples/lite_examples/simple/mnist_example.py @@ -113,7 +113,7 @@ def test(self, model, test_loader): test_loss /= len(test_loader.dataset) - print( + self.print( "\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n".format( test_loss, correct, len(test_loader.dataset), 100.0 * correct / len(test_loader.dataset) ) From 3f7a2ce6a13090fdd703b0669bf81c4eb756a31c Mon Sep 17 00:00:00 2001 From: Kaushik B <45285388+kaushikb11@users.noreply.github.com> Date: Tue, 19 Oct 2021 12:12:41 +0530 Subject: [PATCH 125/331] Update pl_examples/lite_examples/simple/mnist_example.py --- pl_examples/lite_examples/simple/mnist_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/lite_examples/simple/mnist_example.py b/pl_examples/lite_examples/simple/mnist_example.py index 2bd2ccaedb0b0..4990dc506a014 100644 --- a/pl_examples/lite_examples/simple/mnist_example.py +++ b/pl_examples/lite_examples/simple/mnist_example.py @@ -87,7 +87,7 @@ def train(self, args, model, train_loader, optimizer, epoch): self.backward(loss) optimizer.step() if batch_idx % args.log_interval == 0: - print( + self.print( "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format( epoch, batch_idx * len(data), From 61c825c250a5c11bd630754182fe2abb395da2d7 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Tue, 19 Oct 2021 11:21:21 +0100 Subject: [PATCH 126/331] Add LightningLite Example (#9991) Co-authored-by: Kaushik B <45285388+kaushikb11@users.noreply.github.com> --- .../pytorch_2_lite_2_lightning.py | 246 ++++++++++++++++++ .../lite_examples/simple/mnist_example.py | 13 + pytorch_lightning/lite/lite.py | 3 +- 3 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 pl_examples/lite_examples/pytorch_2_lite_2_lightning.py diff --git a/pl_examples/lite_examples/pytorch_2_lite_2_lightning.py b/pl_examples/lite_examples/pytorch_2_lite_2_lightning.py new file mode 100644 index 0000000000000..511031697dc8a --- /dev/null +++ b/pl_examples/lite_examples/pytorch_2_lite_2_lightning.py @@ -0,0 +1,246 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +from torch import nn +from torch.utils.data import DataLoader, Dataset + +from pytorch_lightning import seed_everything +from pytorch_lightning.lite import LightningLite + +############################################################################################# +# Section 1: PyTorch to Lightning Lite # +# # +# What is LightningLite ? # +# # +# `LightningLite` is a python class you can override to get access to Lightning # +# accelerators and scale your training, but furthermore, it is intended to be the safest # +# route to fully transition to Lightning. # +# # +# Does LightningLite requires code changes ? # +# # +# `LightningLite` code changes are minimal and this tutorial will show you how easy it # +# is to convert to `lite` using a `BoringModel`. # +# # +############################################################################################# + +############################################################################################# +# Pure PyTorch Section # +############################################################################################# + + +# 1 / 6: Implement a `BoringModel` with only one layer. +class BoringModel(nn.Module): + def __init__(self): + super().__init__() + self.layer = torch.nn.Linear(32, 2) + + def forward(self, x): + x = self.layer(x) + return torch.nn.functional.mse_loss(x, torch.ones_like(x)) + + +# 2 / 6: Implement a `configure_optimizers` taking a module and returning an optimizer. +def configure_optimizers(module: nn.Module): + return torch.optim.SGD(module.parameters(), lr=0.001) + + +# 3 / 6: Implement a simple dataset returning random data with the specified shape. +class RandomDataset(Dataset): + def __init__(self, length: int, size: int): + self.len = length + self.data = torch.randn(length, size) + + def __getitem__(self, index): + return self.data[index] + + def __len__(self): + return self.len + + +# 4 / 6: Implement the functions to create the dataloaders. +def train_dataloader(): + return DataLoader(RandomDataset(64, 32)) + + +def val_dataloader(): + return DataLoader(RandomDataset(64, 32)) + + +# 5 / 6: Our main PyTorch Loop to train our `BoringModel` on our random data. +def main(model: nn.Module, train_dataloader: DataLoader, val_dataloader: DataLoader, num_epochs: int = 10): + optimizer = configure_optimizers(model) + + for epoch in range(num_epochs): + train_losses = [] + val_losses = [] + + model.train() + for batch in train_dataloader: + optimizer.zero_grad() + loss = model(batch) + train_losses.append(loss) + loss.backward() + optimizer.step() + + model.eval() + with torch.no_grad(): + for batch in val_dataloader: + val_losses.append(model(batch)) + + train_epoch_loss = torch.stack(train_losses).mean() + val_epoch_loss = torch.stack(val_losses).mean() + + print(f"{epoch}/{num_epochs}| Train Epoch Loss: {torch.mean(train_epoch_loss)}") + print(f"{epoch}/{num_epochs}| Valid Epoch Loss: {torch.mean(val_epoch_loss)}") + + return model.state_dict() + + +# 6 / 6: Run the pure PyTorch Loop and train / validate the model. +seed_everything(42) +model = BoringModel() +pure_model_weights = main(model, train_dataloader(), val_dataloader()) + + +############################################################################################# +# Convert to LightningLite # +# # +# By converting to `LightningLite`, you get the full power of Lightning accelerators # +# while conversing your original code ! # +# To get started, you would need to `from pytorch_lightning.lite import LightningLite` # +# and override its `run` method. # +############################################################################################# + + +class LiteTrainer(LightningLite): + def run(self, model: nn.Module, train_dataloader: DataLoader, val_dataloader: DataLoader, num_epochs: int = 10): + optimizer = configure_optimizers(model) + + ################################################################################### + # You would need to call `self.setup` to wrap `model` and `optimizer`. If you # + # have multiple models (c.f GAN), call `setup` for each one of them and their # + # associated optimizers. # + model, optimizer = self.setup(model=model, optimizers=optimizer) + ################################################################################### + + ################################################################################### + # You would need to call `self.setup_dataloaders` to prepare the dataloaders # + # in case you are running in a distributed setting. # + train_dataloader = self.setup_dataloaders(train_dataloader) + val_dataloader = self.setup_dataloaders(val_dataloader) + ################################################################################### + + for epoch in range(num_epochs): + train_losses = [] + val_losses = [] + + model.train() + for batch in train_dataloader: + optimizer.zero_grad() + loss = model(batch) + train_losses.append(loss) + ########################################################################### + # By calling `self.backward` directly, `LightningLite` will automate # + # precision and distributions. # + self.backward(loss) + ########################################################################### + optimizer.step() + + model.eval() + with torch.no_grad(): + for batch in val_dataloader: + val_losses.append(model(batch)) + + train_epoch_loss = torch.stack(train_losses).mean() + val_epoch_loss = torch.stack(val_losses).mean() + + ################################################################################ + # Optional: Utility to print only on rank 0 (when using distributed setting) # + self.print(f"{epoch}/{num_epochs}| Train Epoch Loss: {train_epoch_loss}") + self.print(f"{epoch}/{num_epochs}| Valid Epoch Loss: {val_epoch_loss}") + ################################################################################ + + +seed_everything(42) +lite_model = BoringModel() +lite = LiteTrainer() +lite.run(lite_model, train_dataloader(), val_dataloader()) + +############################################################################################# +# Assert the weights are the same # +############################################################################################# + +for pure_w, lite_w in zip(pure_model_weights.values(), lite_model.state_dict().values()): + torch.equal(pure_w, lite_w) + + +############################################################################################# +# Convert to Lightning # +# # +# By converting to Lightning, not-only your research code becomes inter-operable # +# (can easily be shared), but you get access to hundreds of extra features to make your # +# research faster. # +# Check `Facebook` blogpost on how `Lightning` enabled their research to scale at scale # +# On https://ai.facebook.com/blog # +# /reengineering-facebook-ais-deep-learning-platforms-for-interoperability/ # +############################################################################################# + +from pytorch_lightning import LightningDataModule, LightningModule, Trainer # noqa E402 + + +class LightningBoringModel(LightningModule): + def __init__(self): + super().__init__() + self.layer = torch.nn.Linear(32, 2) + + def forward(self, x): + x = self.layer(x) + return torch.nn.functional.mse_loss(x, torch.ones_like(x)) + + # LightningModule hooks + def training_step(self, batch, batch_idx): + x = self.forward(batch) + self.log("train_loss", x) + return x + + def validation_step(self, batch, batch_idx): + x = self.forward(batch) + self.log("val_loss", x) + return x + + def configure_optimizers(self): + return configure_optimizers(self) + + +class BoringDataModule(LightningDataModule): + def train_dataloader(self): + return train_dataloader() + + def val_dataloader(self): + return val_dataloader() + + +seed_everything(42) +lightning_module = LightningBoringModel() +datamodule = BoringDataModule() +trainer = Trainer(max_epochs=10) +trainer.fit(lightning_module, datamodule) + + +############################################################################################# +# Assert the weights are the same # +############################################################################################# + +for pure_w, lite_w in zip(pure_model_weights.values(), lightning_module.state_dict().values()): + torch.equal(pure_w, lite_w) diff --git a/pl_examples/lite_examples/simple/mnist_example.py b/pl_examples/lite_examples/simple/mnist_example.py index 4990dc506a014..f33271b5cc75b 100644 --- a/pl_examples/lite_examples/simple/mnist_example.py +++ b/pl_examples/lite_examples/simple/mnist_example.py @@ -1,3 +1,16 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import argparse import torch diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 6a78ba46894bc..146542270f8fa 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -135,7 +135,7 @@ def world_size(self) -> int: return getattr(self._strategy, "world_size", 1) @abstractmethod - def run(self, *args: Any, **kwargs: Any) -> None: + def run(self, *args: Any, **kwargs: Any) -> Any: """All the code inside this run method gets accelerated by Lite. Args: @@ -308,7 +308,6 @@ def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> None: self._strategy.spawn(run_method, *args, **kwargs) else: run_method(*args, **kwargs) - # TODO: any teardown needed here? def _set_plugin_specific_precision_variables(self) -> None: # todo: these are hacks as plugins rely on access to the precision plugin From 21ada3644d3d6e131ec3ba911651577fe7edacec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 19 Oct 2021 12:58:11 +0200 Subject: [PATCH 127/331] call process_dataloader() --- pytorch_lightning/lite/lite.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 146542270f8fa..49e9bcbbcafa7 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -226,7 +226,8 @@ def _setup_dataloader( kwargs = TrainerDataLoadingMixin._get_dataloader_init_kwargs(dataloader, sampler) device = self.device if move_to_device else None - return _LiteDataLoader(device=device, **kwargs) + lite_dataloader = _LiteDataLoader(device=device, **kwargs) + return self._strategy.process_dataloader(lite_dataloader) def backward(self, tensor: Tensor, *args: Any, **kwargs: Any) -> None: """Replaces ``loss.backward()`` in your training loop. Handles precision and automatically for you. From 6dd770f5ecde350913d03e25d64244b7db9ad353 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 19 Oct 2021 14:27:19 +0200 Subject: [PATCH 128/331] refactor spawn --- .../plugins/training_type/ddp_spawn.py | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/pytorch_lightning/plugins/training_type/ddp_spawn.py b/pytorch_lightning/plugins/training_type/ddp_spawn.py index 7e5433cb4beba..862f568bea9a8 100644 --- a/pytorch_lightning/plugins/training_type/ddp_spawn.py +++ b/pytorch_lightning/plugins/training_type/ddp_spawn.py @@ -160,26 +160,33 @@ def set_world_ranks(self, process_idx: int = 0) -> None: self.cluster_environment.set_world_size(self.num_nodes * self.num_processes) rank_zero_only.rank = self.cluster_environment.global_rank() - def get_mp_spawn_kwargs(self, trainer: "pl.Trainer") -> dict: - return {"args": (trainer, self.mp_queue), "nprocs": self.num_processes} + def get_mp_spawn_kwargs(self, trainer: Optional["pl.Trainer"] = None) -> Dict[str, Any]: + return {"nprocs": self.num_processes} def start_training(self, trainer: "pl.Trainer") -> None: - # TODO: refactor: call self.spawn() here - mp.spawn(self.new_process, **self.get_mp_spawn_kwargs(trainer)) + self.spawn(self.new_process, trainer, self.mp_queue) # reset optimizers, since main process is never used for training and thus does not have a valid optim state trainer.optimizers = [] def start_evaluating(self, trainer: "pl.Trainer") -> None: - # TODO: refactor: call self.spawn() here - mp.spawn(self.new_process, **self.get_mp_spawn_kwargs(trainer)) + self.spawn(self.new_process, trainer, self.mp_queue) def start_predicting(self, trainer: "pl.Trainer") -> None: - # TODO: refactor: call self.spawn() here - mp.spawn(self.new_process, **self.get_mp_spawn_kwargs(trainer)) + self.spawn(self.new_process, trainer, self.mp_queue) def spawn(self, function: Callable, *args: Any, **kwargs: Any) -> None: + """Spawn processes that run the given function. + + Args: + function: The function to spawn processes from. It must at least accept one positional argument for the + process index. + *args: Optional positional arguments that will be passed to the function in addition to the process index. + These arguments must be pickleable. + **kwargs: Optional named arguments that will be passed to the function in addition to the process index. + These arguments must be pickleable. + """ os.environ["MASTER_PORT"] = str(self.cluster_environment.master_port()) - mp.spawn(self._wrapped_function, args=(function, args, kwargs), nprocs=self.num_processes) + mp.spawn(self._wrapped_function, args=(function, args, kwargs), **self.get_mp_spawn_kwargs()) def _wrapped_function(self, process_idx: int, function: Callable, args: Any, kwargs: Any) -> None: self._worker_setup(process_idx) From 724c2a9bdf5cbe32b7d3e3c3ae377d36931fcf0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 19 Oct 2021 15:06:39 +0200 Subject: [PATCH 129/331] update new_process --- pytorch_lightning/plugins/training_type/ddp_spawn.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pytorch_lightning/plugins/training_type/ddp_spawn.py b/pytorch_lightning/plugins/training_type/ddp_spawn.py index 862f568bea9a8..a8dce866e961e 100644 --- a/pytorch_lightning/plugins/training_type/ddp_spawn.py +++ b/pytorch_lightning/plugins/training_type/ddp_spawn.py @@ -198,8 +198,7 @@ def _worker_setup(self, process_idx: int): rank_zero_only.rank = self.global_rank init_ddp_connection(self.cluster_environment, self.torch_distributed_backend, self.global_rank, self.world_size) - def new_process(self, process_idx: int, trainer: "pl.Trainer", mp_queue: SimpleQueue) -> None: - self._worker_setup(process_idx) + def new_process(self, trainer: "pl.Trainer", mp_queue: SimpleQueue) -> None: self.mp_queue = mp_queue # move the model to the correct device From 866e14d2b16d1d3525b0e2ff02704fef690b5da8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 19 Oct 2021 15:17:34 +0200 Subject: [PATCH 130/331] call accelerator.setup_environment() --- pytorch_lightning/lite/lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index ca2244e78ccd6..1c3b073103820 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -304,7 +304,7 @@ def _run_wrapper(self, run_method: Callable) -> Callable: def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> None: self._set_plugin_specific_precision_variables() - self._strategy.setup_environment() + self._accelerator.setup_environment() if isinstance(self._strategy, DDPSpawnPlugin): self._strategy.spawn(run_method, *args, **kwargs) else: From a2e576bcf6efcdd044e2cbb18a04359bea57d782 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 19 Oct 2021 15:22:12 +0200 Subject: [PATCH 131/331] set_device --- pytorch_lightning/lite/lite.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 1c3b073103820..741768cb38e45 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -261,6 +261,10 @@ def to_device(self, obj: Union[nn.Module, Tensor, Any]) -> Union[nn.Module, Tens A reference to the object that was moved to the new device. """ if isinstance(obj, nn.Module): + if self.device.type == "cuda": + # need to call this manually here again in case we spawned with DDPSpawnPlugin + # TODO: refactor to let plugin handle this cleanly + torch.cuda.set_device(self.device) return obj.to(self.device) return move_data_to_device(obj, device=self.device) From 60050404b9ec0d1823799f4757295d4d8ae50913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 19 Oct 2021 16:42:56 +0200 Subject: [PATCH 132/331] remove unused methods --- pytorch_lightning/plugins/precision/native_amp.py | 11 ----------- .../plugins/precision/precision_plugin.py | 9 --------- 2 files changed, 20 deletions(-) diff --git a/pytorch_lightning/plugins/precision/native_amp.py b/pytorch_lightning/plugins/precision/native_amp.py index 436f5c700a99d..08b8080715d84 100644 --- a/pytorch_lightning/plugins/precision/native_amp.py +++ b/pytorch_lightning/plugins/precision/native_amp.py @@ -101,17 +101,6 @@ def pre_optimizer_step( self.scaler.update() return False - def post_optimizer_step(self, optimizer: "Optimizer", optimizer_idx: int) -> None: - """Updates the GradScaler.""" - self.run_post_optimizer_step(optimizer) - - def run_pre_optimizer_step(self, optimizer: "Optimizer") -> None: - self.scaler.unscale_(optimizer) - - def run_post_optimizer_step(self, optimizer: "Optimizer") -> None: - self.scaler.step(optimizer) - self.scaler.update() - def autocast_context_manager(self) -> torch.cuda.amp.autocast: if self.use_cpu: return torch.cpu.amp.autocast(dtype=self._dtype) # Only reached in pytorch==1.10 where this is ok. skipcq diff --git a/pytorch_lightning/plugins/precision/precision_plugin.py b/pytorch_lightning/plugins/precision/precision_plugin.py index d345fb087d987..dc378e9cb195c 100644 --- a/pytorch_lightning/plugins/precision/precision_plugin.py +++ b/pytorch_lightning/plugins/precision/precision_plugin.py @@ -110,15 +110,6 @@ def pre_optimizer_step( model.trainer.call_hook("on_before_optimizer_step", optimizer, optimizer_idx) return True - def post_optimizer_step(self, optimizer: "Optimizer", optimizer_idx: int) -> None: - """Hook to do something after each optimizer step.""" - - def run_pre_optimizer_step(self, optimizer: "Optimizer"): - """Lightning-independent pre optimizer step logic.""" - - def run_post_optimizer_step(self, optimizer: "Optimizer"): - """Lightning-independent post optimizer step logic.""" - def clip_gradients( self, optimizer: Optimizer, From 6674f015c170dee54925e8d02e645c19f0ca75ca Mon Sep 17 00:00:00 2001 From: Kaushik B <45285388+kaushikb11@users.noreply.github.com> Date: Tue, 19 Oct 2021 20:20:34 +0530 Subject: [PATCH 133/331] Update TPUSpawn plugin to support Lightning Lite (#9999) --- pytorch_lightning/lite/lite.py | 18 +++++++++++-- .../plugins/training_type/tpu_spawn.py | 25 +++++++++++++------ 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 741768cb38e45..657c0a28c9873 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -33,6 +33,7 @@ DDPSpawnPlugin, DeepSpeedPlugin, PLUGIN_INPUT, + TPUSpawnPlugin, TrainingTypePlugin, ) from pytorch_lightning.trainer.connectors.accelerator_connector import AcceleratorConnector @@ -164,7 +165,17 @@ def setup( optimizers = [optimizers] if isinstance(optimizers, Optimizer) else optimizers if move_to_device: + params_on_cpu = dict(model.named_parameters()) model = self.to_device(model) + params_on_device = dict(model.named_parameters()) + + # When the user creates the optimizer, they reference the parameters on the CPU. + # However, when running with TPU the parameters get copied and the reference in the optimizer + # remains invalid. We need to update the references to point to the parameter tensors on the device. + mapping = {param: params_on_device[name] for name, param in params_on_cpu.items()} + for optimizer in optimizers: + for param_group in optimizer.param_groups: + param_group["params"] = [mapping.get(p, p) for p in param_group["params"]] model, optimizers = self._setup_model_and_optimizers(model, optimizers) optimizers = optimizers[0] if len(optimizers) == 1 else optimizers @@ -226,8 +237,11 @@ def _setup_dataloader( kwargs = TrainerDataLoadingMixin._get_dataloader_init_kwargs(dataloader, sampler) device = self.device if move_to_device else None - lite_dataloader = _LiteDataLoader(device=device, **kwargs) - return self._strategy.process_dataloader(lite_dataloader) + if isinstance(self._strategy, TPUSpawnPlugin): + dataloader = DataLoader(**kwargs) + else: + dataloader = _LiteDataLoader(device=device, **kwargs) + return self._strategy.process_dataloader(dataloader) def backward(self, tensor: Tensor, *args: Any, **kwargs: Any) -> None: """Replaces ``loss.backward()`` in your training loop. Handles precision and automatically for you. diff --git a/pytorch_lightning/plugins/training_type/tpu_spawn.py b/pytorch_lightning/plugins/training_type/tpu_spawn.py index 73f7467f575cf..3c17c92461cac 100644 --- a/pytorch_lightning/plugins/training_type/tpu_spawn.py +++ b/pytorch_lightning/plugins/training_type/tpu_spawn.py @@ -150,16 +150,9 @@ def set_world_ranks(self, process_idx: int = 0) -> None: pass def new_process(self, process_idx: int, trainer, mp_queue) -> None: + self._worker_setup(process_idx) self.mp_queue = mp_queue - reset_seed() - - self.tpu_local_core_rank = xm.get_local_ordinal() - self.tpu_global_core_rank = xm.get_ordinal() - - # set warning rank - rank_zero_only.rank = self.global_rank - if self.tpu_global_core_rank != 0 and trainer.progress_bar_callback is not None: trainer.progress_bar_callback.disable() @@ -192,6 +185,9 @@ def new_process(self, process_idx: int, trainer, mp_queue) -> None: def model_to_device(self) -> None: self.model = self.wrapped_model.to(self.root_device) + def setup_model(self, model: Module) -> Module: + return model + def barrier(self, name: Optional[str] = None) -> None: if self.is_distributed: rendezvous(name) @@ -272,6 +268,19 @@ def get_mp_spawn_kwargs(self, trainer: "pl.Trainer") -> dict: def optimizer_step(self, optimizer: Optimizer, lambda_closure: Callable, **kwargs): xm.optimizer_step(optimizer, barrier=False, optimizer_args={"closure": lambda_closure, **kwargs}) + def spawn(self, function: Callable, *args: Any, **kwargs: Any) -> None: + xmp.spawn(self._wrapped_function, args=(function, args, kwargs), nprocs=self.num_processes) + + def _wrapped_function(self, process_idx: int, function: Callable, args: Any, kwargs: Any) -> None: + self._worker_setup(process_idx) + function(*args, **kwargs) + + def _worker_setup(self, process_idx: int): + reset_seed() + self.tpu_local_core_rank = xm.get_local_ordinal() + self.tpu_global_core_rank = xm.get_ordinal() + rank_zero_only.rank = self.global_rank + def start_training(self, trainer: "pl.Trainer") -> None: # todo: precision pluging is call in accelerator setup and should be moved if "XLA_USE_BF16" in os.environ: From 5bdde6290ba7b6a693a9017e96b4373bf9fdf766 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 19 Oct 2021 17:42:47 +0200 Subject: [PATCH 134/331] move tpu optimizer step --- pytorch_lightning/accelerators/tpu.py | 3 --- pytorch_lightning/plugins/training_type/tpu_spawn.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pytorch_lightning/accelerators/tpu.py b/pytorch_lightning/accelerators/tpu.py index bd47fd11a5591..0628edf340326 100644 --- a/pytorch_lightning/accelerators/tpu.py +++ b/pytorch_lightning/accelerators/tpu.py @@ -49,9 +49,6 @@ def setup(self, trainer: "pl.Trainer") -> None: raise MisconfigurationException("TPUs only support a single tpu core or tpu spawn training.") return super().setup(trainer) - def run_optimizer_step(self, optimizer: Optimizer, lambda_closure: Callable, **kwargs: Any) -> None: - xm.optimizer_step(optimizer, optimizer_args={"closure": lambda_closure, **kwargs}) - def _move_optimizer_state(self, device: Optional[torch.device] = None) -> None: """Moves the state of the optimizers to the TPU if needed.""" # TODO: `self.root_device` would raise error if called outside the spawn process diff --git a/pytorch_lightning/plugins/training_type/tpu_spawn.py b/pytorch_lightning/plugins/training_type/tpu_spawn.py index 3c17c92461cac..a5da1abdab624 100644 --- a/pytorch_lightning/plugins/training_type/tpu_spawn.py +++ b/pytorch_lightning/plugins/training_type/tpu_spawn.py @@ -265,7 +265,7 @@ def get_mp_spawn_kwargs(self, trainer: "pl.Trainer") -> dict: "start_method": self.start_method, } - def optimizer_step(self, optimizer: Optimizer, lambda_closure: Callable, **kwargs): + def optimizer_step(self, optimizer: Optimizer, lambda_closure: Callable, **kwargs) -> None: xm.optimizer_step(optimizer, barrier=False, optimizer_args={"closure": lambda_closure, **kwargs}) def spawn(self, function: Callable, *args: Any, **kwargs: Any) -> None: From 9f4943c1a688f2af527484b9bf714214076ef136 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 19 Oct 2021 20:55:55 +0200 Subject: [PATCH 135/331] clean up imports --- pytorch_lightning/plugins/training_type/ddp.py | 1 - pytorch_lightning/plugins/training_type/ddp_spawn.py | 1 - pytorch_lightning/plugins/training_type/dp.py | 2 +- pytorch_lightning/utilities/distributed.py | 1 + 4 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pytorch_lightning/plugins/training_type/ddp.py b/pytorch_lightning/plugins/training_type/ddp.py index 61442b275bcfa..64fc1a5a97277 100644 --- a/pytorch_lightning/plugins/training_type/ddp.py +++ b/pytorch_lightning/plugins/training_type/ddp.py @@ -29,7 +29,6 @@ import torch.distributed from torch.nn import Module from torch.nn.parallel.distributed import DistributedDataParallel -from torch.utils.data import DataLoader, DistributedSampler import pytorch_lightning as pl from pytorch_lightning.core.optimizer import LightningOptimizer diff --git a/pytorch_lightning/plugins/training_type/ddp_spawn.py b/pytorch_lightning/plugins/training_type/ddp_spawn.py index 0da52be3e73ed..c72cc7f31d0cc 100644 --- a/pytorch_lightning/plugins/training_type/ddp_spawn.py +++ b/pytorch_lightning/plugins/training_type/ddp_spawn.py @@ -14,7 +14,6 @@ import logging import os import re -from functools import partial from multiprocessing.queues import SimpleQueue from typing import Any, Callable, Dict, List, Optional, Union diff --git a/pytorch_lightning/plugins/training_type/dp.py b/pytorch_lightning/plugins/training_type/dp.py index f83e8240c263f..a78ba2d0b38a6 100644 --- a/pytorch_lightning/plugins/training_type/dp.py +++ b/pytorch_lightning/plugins/training_type/dp.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Optional, Sequence +from typing import List, Optional import torch from torch.nn import DataParallel, Module diff --git a/pytorch_lightning/utilities/distributed.py b/pytorch_lightning/utilities/distributed.py index 76dafbaadeaeb..3db371b252490 100644 --- a/pytorch_lightning/utilities/distributed.py +++ b/pytorch_lightning/utilities/distributed.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import logging import os from functools import wraps From 365fc8da139d628eba15f4492c059b04861bf74e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 19 Oct 2021 20:57:32 +0200 Subject: [PATCH 136/331] remove distributed_backend --- pytorch_lightning/lite/lite.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 657c0a28c9873..051722a24f9f7 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -85,7 +85,6 @@ def __init__( devices=devices, tpu_cores=tpu_cores, ipus=None, - distributed_backend=None, accelerator=accelerator, strategy=strategy, gpus=gpus, From f63b85e71eb8883c80a549ea8e704f6b2033fb72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 19 Oct 2021 20:58:47 +0200 Subject: [PATCH 137/331] access protected methods --- pytorch_lightning/lite/lite.py | 2 +- pytorch_lightning/plugins/training_type/sharded.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 051722a24f9f7..c57aeb64dbe8d 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -346,7 +346,7 @@ def _setup_model_and_optimizers( optimizers: List[Optimizer], ) -> Tuple[_LiteModule, List[_LiteOptimizer]]: # Let accelerator/plugin wrap and connect the models and optimizers - [model], optimizers = self._strategy.setup_models_and_optimizers([model], optimizers) + [model], optimizers = self._strategy._setup_models_and_optimizers([model], optimizers) model = _LiteModule(module=model, accelerator=self._accelerator) optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self._accelerator) for optimizer in optimizers] return model, optimizers diff --git a/pytorch_lightning/plugins/training_type/sharded.py b/pytorch_lightning/plugins/training_type/sharded.py index 099f7a24bfd0a..8b0473b25c215 100644 --- a/pytorch_lightning/plugins/training_type/sharded.py +++ b/pytorch_lightning/plugins/training_type/sharded.py @@ -41,7 +41,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._precision = None - def setup_models_and_optimizers( + def _setup_models_and_optimizers( self, models: List[Module], optimizers: List[Optimizer] ) -> Tuple[List[Module], List[Optimizer]]: if len(models) > 1: @@ -59,7 +59,7 @@ def configure_ddp(self) -> None: # For multi-node training, enabling bucketing will improve performance. self._ddp_kwargs["reduce_buffer_size"] = self._REDUCE_BUFFER_SIZE_DEFAULT if self.num_nodes > 1 else 0 - [self._model], optimizers = self.setup_models_and_optimizers( + [self._model], optimizers = self._setup_models_and_optimizers( models=[LightningShardedDataParallel(self.model)], optimizers=self.lightning_module.trainer.optimizers, ) From 2b77ce2ad9e2310cfae8dbbb9c5987734039e402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 19 Oct 2021 21:10:34 +0200 Subject: [PATCH 138/331] update precision handling in sharded --- pytorch_lightning/plugins/training_type/sharded.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pytorch_lightning/plugins/training_type/sharded.py b/pytorch_lightning/plugins/training_type/sharded.py index 8b0473b25c215..b749e7ca9f5fc 100644 --- a/pytorch_lightning/plugins/training_type/sharded.py +++ b/pytorch_lightning/plugins/training_type/sharded.py @@ -75,9 +75,7 @@ def _reinit_optimizers_with_oss(self, optimizers: List[Union[Optimizer, Lightnin optim_class = type(optimizer) zero_optimizer = OSS(params=optimizer.param_groups, optim=optim_class, **optimizer.defaults) if _FAIRSCALE_OSS_FP16_BROADCAST_AVAILABLE: - precision = self._precision or ( - 32 if self.lightning_module is None else self.lightning_module.trainer.precision - ) + precision = self._precision or self.lightning_module.trainer.precision is_fp16 = precision in ("mixed", 16) # For multi-node training, compressing the model shards in fp16 before broadcasting # improves performance. When using PyTorch AMP, it will not degrade From db00696d9add2f633da5906e174da85e69e01144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 19 Oct 2021 21:39:08 +0200 Subject: [PATCH 139/331] sharded spawn --- .../plugins/training_type/sharded.py | 27 ++++++------ .../plugins/training_type/sharded_spawn.py | 43 ++++++++++++------- 2 files changed, 42 insertions(+), 28 deletions(-) diff --git a/pytorch_lightning/plugins/training_type/sharded.py b/pytorch_lightning/plugins/training_type/sharded.py index b749e7ca9f5fc..2640eacf4fb63 100644 --- a/pytorch_lightning/plugins/training_type/sharded.py +++ b/pytorch_lightning/plugins/training_type/sharded.py @@ -41,19 +41,6 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._precision = None - def _setup_models_and_optimizers( - self, models: List[Module], optimizers: List[Optimizer] - ) -> Tuple[List[Module], List[Optimizer]]: - if len(models) > 1: - raise ValueError( - f"DDPSharded only supports a single model with one or several optimizers. Got {len(models)} models." - ) - - optimizers = self._wrap_optimizers(optimizers) - model = ShardedDataParallel(models[0], sharded_optimizer=optimizers, **self._ddp_kwargs) - setattr(model, "require_backward_grad_sync", False) # TODO: needed? - return [model], optimizers - def configure_ddp(self) -> None: if "reduce_buffer_size" not in self._ddp_kwargs: # For multi-node training, enabling bucketing will improve performance. @@ -67,6 +54,20 @@ def configure_ddp(self) -> None: trainer.optimizers = optimizers trainer.convert_to_lightning_optimizers() + def _setup_models_and_optimizers( + self, models: List[Module], optimizers: List[Optimizer] + ) -> Tuple[List[Module], List[Optimizer]]: + if len(models) > 1: + raise ValueError( + f"DDPSharded only supports setting up a single model with one or several optimizers." + f" Got {len(models)} models." + ) + + optimizers = self._wrap_optimizers(optimizers) + model = ShardedDataParallel(models[0], sharded_optimizer=optimizers, **self._ddp_kwargs) + setattr(model, "require_backward_grad_sync", False) # TODO: needed? + return [model], optimizers + def _reinit_optimizers_with_oss(self, optimizers: List[Union[Optimizer, LightningOptimizer]]) -> List["OSS"]: for x, optimizer in enumerate(optimizers): if isinstance(optimizer, LightningOptimizer): diff --git a/pytorch_lightning/plugins/training_type/sharded_spawn.py b/pytorch_lightning/plugins/training_type/sharded_spawn.py index 78b54d029a5f6..6b49dd80bcb82 100644 --- a/pytorch_lightning/plugins/training_type/sharded_spawn.py +++ b/pytorch_lightning/plugins/training_type/sharded_spawn.py @@ -13,9 +13,11 @@ # limitations under the License. from contextlib import contextmanager from multiprocessing.queues import SimpleQueue -from typing import Dict, Generator, Optional +from typing import Dict, Generator, Optional, List, Tuple import torch +from torch.nn import Module +from torch.optim import Optimizer import pytorch_lightning as pl from pytorch_lightning.plugins.precision.sharded_native_amp import ShardedNativeMixedPrecisionPlugin @@ -36,29 +38,40 @@ class DDPSpawnShardedPlugin(DDPSpawnPlugin): """Optimizer sharded training provided by FairScale.""" def configure_ddp(self) -> None: - self._wrap_optimizers() - self._model = ShardedDataParallel( - LightningShardedDataParallel(self.model), - sharded_optimizer=self.lightning_module.trainer.optimizers, - **self._ddp_kwargs + [self._model], optimizers = self._setup_models_and_optimizers( + models=[LightningShardedDataParallel(self.model)], + optimizers=self.lightning_module.trainer.optimizers, ) - setattr(self._model, "require_backward_grad_sync", False) + self.lightning_module.trainer.optimizers = optimizers + + def _setup_models_and_optimizers( + self, models: List[Module], optimizers: List[Optimizer] + ) -> Tuple[List[Module], List[Optimizer]]: + if len(models) > 1: + raise ValueError( + f"DDPShardedSpawn only supports setting up a single model with one or several optimizers." + f" Got {len(models)} models." + ) + + optimizers = self._wrap_optimizers(optimizers) + model = ShardedDataParallel(models[0], sharded_optimizer=optimizers, **self._ddp_kwargs) + setattr(model, "require_backward_grad_sync", False) # TODO: needed? + return [model], optimizers - def _reinit_optimizers_with_oss(self): - optimizers = self.lightning_module.trainer.optimizers + def _reinit_optimizers_with_oss(self, optimizers: List[Optimizer]) -> List["OSS"]: for x, optimizer in enumerate(optimizers): if not isinstance(optimizer, OSS): optim_class = type(optimizer) zero_optimizer = OSS(params=optimizer.param_groups, optim=optim_class, **optimizer.defaults) optimizers[x] = zero_optimizer del optimizer - trainer = self.lightning_module.trainer - trainer.optimizers = optimizers + return optimizers + + def _wrap_optimizers(self, optimizers: List[Optimizer]) -> List["OSS"]: + if self.model is not None and self.model.trainer.state.fn != TrainerFn.FITTING: + return optimizers - def _wrap_optimizers(self): - if self.model.trainer.state.fn != TrainerFn.FITTING: - return - self._reinit_optimizers_with_oss() + return self._reinit_optimizers_with_oss(optimizers) def optimizer_state(self, optimizer: "OSS") -> Optional[dict]: if isinstance(optimizer, OSS): From 71a64892bf17b62ecd8be205ca94c583704d4dcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Tue, 19 Oct 2021 21:41:29 +0200 Subject: [PATCH 140/331] add spawn shaded support --- pytorch_lightning/lite/lite.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index c57aeb64dbe8d..8d70ff2ffec2d 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -401,4 +401,5 @@ def _supported_strategy_types() -> Sequence[str]: DistributedType.DP, DistributedType.DEEPSPEED, DistributedType.DDP_SHARDED, + DistributedType.DDP_SHARDED_SPAWN, ) From 0a06e8554e8a5a6ecf70a862fcd62e721f472736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 20 Oct 2021 01:08:10 +0200 Subject: [PATCH 141/331] add zero grad stub to LiteOptimizer --- pytorch_lightning/lite/wrappers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index c6e3c6a113880..a37b16c0654d9 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -65,6 +65,9 @@ def step(self, closure: Optional[Callable] = None) -> None: model=None, ) + def zero_grad(self, set_to_none: bool = False) -> None: + self._optimizer.zero_grad(set_to_none=set_to_none) + class _LiteModule(nn.Module): def __init__(self, module: nn.Module, accelerator: Accelerator) -> None: From a488fd9f8c1c4b6a0d64a3bedab1d10e2ff193ba Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 20 Oct 2021 05:00:05 +0000 Subject: [PATCH 142/331] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytorch_lightning/plugins/training_type/sharded_spawn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/plugins/training_type/sharded_spawn.py b/pytorch_lightning/plugins/training_type/sharded_spawn.py index 6b49dd80bcb82..6f6fec9496636 100644 --- a/pytorch_lightning/plugins/training_type/sharded_spawn.py +++ b/pytorch_lightning/plugins/training_type/sharded_spawn.py @@ -13,7 +13,7 @@ # limitations under the License. from contextlib import contextmanager from multiprocessing.queues import SimpleQueue -from typing import Dict, Generator, Optional, List, Tuple +from typing import Dict, Generator, List, Optional, Tuple import torch from torch.nn import Module From 072fff0506d143cd47a9425dffb59be932cf88a6 Mon Sep 17 00:00:00 2001 From: Kaushik B Date: Wed, 20 Oct 2021 13:22:00 +0530 Subject: [PATCH 143/331] Remove TPUAcc check in setup_dataloaders --- pytorch_lightning/lite/lite.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 8d70ff2ffec2d..dbcc02e8b71d5 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -26,7 +26,7 @@ from torch.utils.data import DataLoader, DistributedSampler, RandomSampler, SequentialSampler from pytorch_lightning import Trainer -from pytorch_lightning.accelerators import Accelerator, TPUAccelerator +from pytorch_lightning.accelerators import Accelerator from pytorch_lightning.lite.wrappers import _LiteDataLoader, _LiteModule, _LiteOptimizer from pytorch_lightning.plugins import ( DDPShardedPlugin, @@ -222,9 +222,7 @@ def _setup_dataloader( The wrapped dataloader. """ sampler = dataloader.sampler - if replace_sampler and ( - self._requires_distributed_sampler(dataloader) or isinstance(self._accelerator, TPUAccelerator) - ): + if replace_sampler and self._requires_distributed_sampler(dataloader): if not isinstance(dataloader.sampler, (SequentialSampler, RandomSampler)): raise MisconfigurationException( "You seem to have configured a sampler in your DataLoader. This will be replaced " From cd2f0d638fc0fbb4448905df71a2d5029dc69cd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 20 Oct 2021 10:10:11 +0200 Subject: [PATCH 144/331] trigger ci --- pl_examples/lite_examples/simple/mnist_example.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pl_examples/lite_examples/simple/mnist_example.py b/pl_examples/lite_examples/simple/mnist_example.py index f33271b5cc75b..3158170bce057 100644 --- a/pl_examples/lite_examples/simple/mnist_example.py +++ b/pl_examples/lite_examples/simple/mnist_example.py @@ -52,6 +52,8 @@ def forward(self, x): class MNIST(LightningLite): + + # when we enter run() here, distributed setup already took place def run(self, args): use_cuda = self.device.type == "cuda" From 4ed755be41a2bbff60f8be355f463ba7b3be0608 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Wed, 20 Oct 2021 10:20:50 +0100 Subject: [PATCH 145/331] Add parity tests for LightningLite vs. pure PyTorch (#10002) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrian Wälchli --- pytorch_lightning/lite/lite.py | 6 +- .../plugins/training_type/ddp_spawn.py | 17 +- tests/lite/__init__.py | 0 tests/lite/test_parity.py | 233 ++++++++++++++++++ 4 files changed, 248 insertions(+), 8 deletions(-) create mode 100644 tests/lite/__init__.py create mode 100644 tests/lite/test_parity.py diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index dbcc02e8b71d5..b16ee73dba769 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -317,13 +317,13 @@ def execute_on_rank(self, func: Callable, rank: int, *args: Any, **kwargs: Any) def _run_wrapper(self, run_method: Callable) -> Callable: return partial(self._run_impl, run_method) - def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> None: + def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> Any: self._set_plugin_specific_precision_variables() self._accelerator.setup_environment() if isinstance(self._strategy, DDPSpawnPlugin): - self._strategy.spawn(run_method, *args, **kwargs) + return self._strategy.spawn(run_method, *args, **kwargs) else: - run_method(*args, **kwargs) + return run_method(*args, **kwargs) def _set_plugin_specific_precision_variables(self) -> None: # todo: these are hacks as plugins rely on access to the precision plugin diff --git a/pytorch_lightning/plugins/training_type/ddp_spawn.py b/pytorch_lightning/plugins/training_type/ddp_spawn.py index c72cc7f31d0cc..1c9a654bc0b9e 100644 --- a/pytorch_lightning/plugins/training_type/ddp_spawn.py +++ b/pytorch_lightning/plugins/training_type/ddp_spawn.py @@ -38,7 +38,7 @@ rank_zero_deprecation, rank_zero_warn, ) -from pytorch_lightning.utilities.apply_func import apply_to_collection +from pytorch_lightning.utilities.apply_func import apply_to_collection, move_data_to_device from pytorch_lightning.utilities.cloud_io import atomic_save from pytorch_lightning.utilities.cloud_io import load as pl_load from pytorch_lightning.utilities.distributed import distributed_available @@ -174,7 +174,7 @@ def start_evaluating(self, trainer: "pl.Trainer") -> None: def start_predicting(self, trainer: "pl.Trainer") -> None: self.spawn(self.new_process, trainer, self.mp_queue) - def spawn(self, function: Callable, *args: Any, **kwargs: Any) -> None: + def spawn(self, function: Callable, *args: Any, **kwargs: Any) -> Any: """Spawn processes that run the given function. Args: @@ -185,11 +185,18 @@ def spawn(self, function: Callable, *args: Any, **kwargs: Any) -> None: These arguments must be pickleable. """ os.environ["MASTER_PORT"] = str(self.cluster_environment.master_port()) - mp.spawn(self._wrapped_function, args=(function, args, kwargs), **self.get_mp_spawn_kwargs()) + smp = mp.get_context("spawn") + return_queue = smp.SimpleQueue() + mp.spawn(self._wrapped_function, args=(function, args, kwargs, return_queue), nprocs=self.num_processes) + return return_queue.get() - def _wrapped_function(self, process_idx: int, function: Callable, args: Any, kwargs: Any) -> None: + def _wrapped_function( + self, process_idx: int, function: Callable, args: Any, kwargs: Any, return_queue: SimpleQueue + ) -> None: self._worker_setup(process_idx) - function(*args, **kwargs) + result = function(*args, **kwargs) + if self.is_global_zero: + return_queue.put(move_data_to_device(result, "cpu")) def _worker_setup(self, process_idx: int): reset_seed() diff --git a/tests/lite/__init__.py b/tests/lite/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/lite/test_parity.py b/tests/lite/test_parity.py new file mode 100644 index 0000000000000..57657bedb5cc1 --- /dev/null +++ b/tests/lite/test_parity.py @@ -0,0 +1,233 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from contextlib import contextmanager +from copy import deepcopy +from functools import partial +from typing import Callable, Generator + +import pytest +import torch +import torch.distributed +import torch.multiprocessing as mp +import torch.nn.functional +from torch import nn +from torch.nn.parallel.distributed import DistributedDataParallel +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler + +from pytorch_lightning import seed_everything +from pytorch_lightning.lite import LightningLite +from pytorch_lightning.plugins.environments.lightning_environment import find_free_network_port +from pytorch_lightning.plugins.training_type.ddp_spawn import DDPSpawnPlugin +from pytorch_lightning.utilities.apply_func import apply_to_collection, move_data_to_device +from pytorch_lightning.utilities.cloud_io import atomic_save +from pytorch_lightning.utilities.imports import _TORCH_BFLOAT_AVAILABLE +from tests.helpers.boring_model import RandomDataset +from tests.helpers.runif import RunIf + + +class BoringModel(nn.Module): + def __init__(self): + super().__init__() + self.layer = torch.nn.Linear(32, 2, bias=False) + + def forward(self, x): + x = self.layer(x) + return torch.nn.functional.mse_loss(x, torch.ones_like(x)) + + +def configure_optimizers(module: nn.Module): + return torch.optim.SGD(module.parameters(), lr=0.0001) + + +def main( + move_to_device: Callable, + model: nn.Module, + train_dataloader: DataLoader, + num_epochs: int = 10, +): + model = move_to_device(model) + optimizer = configure_optimizers(model) + + for _ in range(num_epochs): + model.train() + for batch in train_dataloader: + batch = move_to_device(batch) + optimizer.zero_grad() + loss = model(batch) + loss.backward() + optimizer.step() + + return model.state_dict() + + +class LiteRunner(LightningLite): + def run(self, model: nn.Module, train_dataloader: DataLoader, num_epochs: int = 10, tmpdir: str = None): + optimizer = configure_optimizers(model) + model, optimizer = self.setup(model=model, optimizers=optimizer) + train_dataloader = self.setup_dataloaders(train_dataloader) + + model.train() + for _ in range(num_epochs): + for batch in train_dataloader: + batch = self.to_device(batch) + optimizer.zero_grad() + loss = model(batch) + self.backward(loss) + optimizer.step() + + if isinstance(self._strategy, DDPSpawnPlugin) and tmpdir and self.global_rank == 0: + checkpoint_path = os.path.join(tmpdir, "model.pt") + atomic_save(model.state_dict(), checkpoint_path) + return checkpoint_path + + +@contextmanager +def precision_context(precision, accelerator) -> Generator[None, None, None]: + if precision == 32: + yield + return + if precision == 16 and accelerator == "gpu": + with torch.cuda.amp.autocast(): + yield + elif accelerator == "cpu": + with torch.cpu.amp.autocast(dtype=torch.float16 if precision == 16 else torch.bfloat16): + yield + else: + with torch.cuda.amp.autocast(): + yield + + +@pytest.mark.parametrize( + "precision, strategy, devices, accelerator", + [ + pytest.param(32, None, 1, "cpu"), + pytest.param(32, None, 1, "gpu", marks=RunIf(min_gpus=1)), + pytest.param(16, None, 1, "gpu", marks=RunIf(min_gpus=1)), + pytest.param( + "bf16", + None, + 1, + "gpu", + marks=pytest.mark.skipif(not _TORCH_BFLOAT_AVAILABLE, reason="bfloat16 isn't available."), + ), + ], +) +def test_boring_lite_model_single_device(precision, strategy, devices, accelerator, tmpdir): + seed_everything(42) + train_dataloader = DataLoader(RandomDataset(32, 8)) + model = BoringModel() + num_epochs = 1 + state_dict = deepcopy(model.state_dict()) + + lite = LiteRunner(precision=precision, strategy=strategy, devices=devices, accelerator=accelerator) + lite.run(model, train_dataloader, num_epochs=num_epochs) + lite_state_dict = model.state_dict() + + with precision_context(precision, accelerator): + model.load_state_dict(state_dict) + pure_state_dict = main(lite.to_device, model, train_dataloader, num_epochs=num_epochs) + + state_dict = apply_to_collection(state_dict, torch.Tensor, lite.to_device) + for w_pure, w_lite in zip(state_dict.values(), lite_state_dict.values()): + assert not torch.equal(w_pure, w_lite) + + for w_pure, w_lite in zip(pure_state_dict.values(), lite_state_dict.values()): + assert torch.equal(w_pure, w_lite) + + +def run(rank, model, train_dataloader, num_epochs, precision, accelerator, tmpdir): + os.environ["LOCAL_RANK"] = str(rank) + if torch.distributed.is_available() and not torch.distributed.is_initialized(): + torch.distributed.init_process_group("gloo", rank=rank, world_size=2) + + to_device = partial(move_data_to_device, device=torch.device("cuda", rank)) + model = DistributedDataParallel( + to_device(model), + device_ids=[rank], + ) + train_dataloader = DataLoader( + train_dataloader.dataset, + sampler=DistributedSampler(train_dataloader.dataset, rank=rank, num_replicas=2, seed=42, drop_last=False), + ) + with precision_context(precision, accelerator): + main(to_device, model, train_dataloader, num_epochs=num_epochs) + + if rank == 0: + atomic_save(model.state_dict(), os.path.join(tmpdir, "model_spawn.pt")) + + +# @pytest.mark.skipif(True, reason="Skipping as it takes 80 seconds.") +@RunIf(min_gpus=2) +@pytest.mark.parametrize( + "precision, strategy, devices, accelerator", + [ + (32, "ddp_spawn", 2, "gpu"), + ], +) +def test_boring_lite_model_ddp_spawn(precision, strategy, devices, accelerator, tmpdir): + seed_everything(42) + train_dataloader = DataLoader(RandomDataset(32, 8)) + model = BoringModel() + num_epochs = 1 + state_dict = deepcopy(model.state_dict()) + + lite = LiteRunner(precision=precision, strategy=strategy, devices=devices, accelerator=accelerator) + checkpoint_path = lite.run(model, train_dataloader, num_epochs=num_epochs, tmpdir=tmpdir) + spawn_model_state_dict = torch.load(checkpoint_path) + + for w_pure, w_lite in zip(state_dict.values(), spawn_model_state_dict.values()): + assert not torch.equal(w_pure.cpu(), w_lite.cpu()) + + model.load_state_dict(state_dict) + os.environ["MASTER_ADDR"] = "127.0.0.1" + os.environ["MASTER_PORT"] = str(find_free_network_port()) + mp.spawn(run, args=(model, train_dataloader, num_epochs, precision, accelerator, tmpdir), nprocs=2) + spawn_pure_model_state_dict = torch.load(os.path.join(tmpdir, "model_spawn.pt")) + + for w_pure, w_lite in zip(spawn_pure_model_state_dict.values(), spawn_model_state_dict.values()): + assert torch.equal(w_pure.cpu(), w_lite.cpu()) + + +@RunIf(min_gpus=2, special=True) +@pytest.mark.parametrize( + "precision, strategy, devices, accelerator", + [ + (32, "ddp", 2, "gpu"), + ], +) +def test_boring_lite_model_ddp(precision, strategy, devices, accelerator, tmpdir): + seed_everything(42) + train_dataloader = DataLoader(RandomDataset(32, 4)) + model = BoringModel() + num_epochs = 1 + state_dict = deepcopy(model.state_dict()) + + lite = LiteRunner(precision=precision, strategy=strategy, devices=devices, accelerator=accelerator) + lite.run(model, train_dataloader, num_epochs=num_epochs, tmpdir=tmpdir) + + lite_model_state_dict = model.state_dict() + + for w_pure, w_lite in zip(state_dict.values(), lite_model_state_dict.values()): + assert not torch.equal(w_pure.cpu(), w_lite.cpu()) + + seed_everything(42) + train_dataloader = DataLoader(RandomDataset(32, 4)) + model = BoringModel() + run(lite.global_rank, model, train_dataloader, num_epochs, precision, accelerator, tmpdir) + pure_model_state_dict = model.state_dict() + + for w_pure, w_lite in zip(pure_model_state_dict.values(), lite_model_state_dict.values()): + assert torch.equal(w_pure.cpu(), w_lite.cpu()) From 75100ad01911a4e59a40b4562033545711847443 Mon Sep 17 00:00:00 2001 From: tchaton Date: Wed, 20 Oct 2021 12:01:01 +0100 Subject: [PATCH 146/331] update test --- tests/lite/test_parity.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/lite/test_parity.py b/tests/lite/test_parity.py index 57657bedb5cc1..5efd9bae63914 100644 --- a/tests/lite/test_parity.py +++ b/tests/lite/test_parity.py @@ -23,6 +23,7 @@ import torch.multiprocessing as mp import torch.nn.functional from torch import nn +from torch.cuda import is_available from torch.nn.parallel.distributed import DistributedDataParallel from torch.utils.data import DataLoader from torch.utils.data.distributed import DistributedSampler @@ -114,14 +115,17 @@ def precision_context(precision, accelerator) -> Generator[None, None, None]: "precision, strategy, devices, accelerator", [ pytest.param(32, None, 1, "cpu"), - pytest.param(32, None, 1, "gpu", marks=RunIf(min_gpus=1)), - pytest.param(16, None, 1, "gpu", marks=RunIf(min_gpus=1)), + pytest.param(32, None, 1, "gpu", marks=pytest.mark.skipif(not is_available(), reason="requires a GPU")), + pytest.param(16, None, 1, "gpu", marks=pytest.mark.skipif(not is_available(), reason="requires a GPU")), pytest.param( "bf16", None, 1, "gpu", - marks=pytest.mark.skipif(not _TORCH_BFLOAT_AVAILABLE, reason="bfloat16 isn't available."), + marks=pytest.mark.skipif( + not (_TORCH_BFLOAT_AVAILABLE and is_available()), + reason="bfloat16 and requires GPU isn't available.", + ), ), ], ) From 72b47cd33fc998d6793e0a13df3f905744f6e22b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 20 Oct 2021 14:25:30 +0200 Subject: [PATCH 147/331] Add tests for Lite wrappers (#10048) --- pytorch_lightning/lite/wrappers.py | 10 +-- tests/lite/test_wrappers.py | 106 +++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 5 deletions(-) create mode 100644 tests/lite/test_wrappers.py diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index a37b16c0654d9..90924be2f23b6 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -82,17 +82,17 @@ def module(self) -> nn.Module: def forward(self, *args: Any, **kwargs: Any) -> Any: precision = self._accelerator.precision_plugin.precision precision_to_type = { - "mixed": torch.half, - 16: torch.half, - 32: torch.float, - 64: torch.double, + "mixed": torch.float16, + 16: torch.float16, + 32: torch.float32, + 64: torch.float64, } # TODO (@awaelchli): let the precision plugin handle the conversion to_type = precision_to_type[precision] args, kwargs = apply_to_collection([args, kwargs], function=lambda t: t.to(to_type), dtype=Tensor) with self._accelerator.precision_plugin.forward_context(): - output = self.module.forward(*args, **kwargs) + output = self.module(*args, **kwargs) output = apply_to_collection(output, function=lambda t: t.to(torch.get_default_dtype()), dtype=Tensor) return output diff --git a/tests/lite/test_wrappers.py b/tests/lite/test_wrappers.py new file mode 100644 index 0000000000000..c58a750afaf03 --- /dev/null +++ b/tests/lite/test_wrappers.py @@ -0,0 +1,106 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from unittest.mock import Mock + +import pytest +import torch + +from pytorch_lightning.lite import LightningLite +from pytorch_lightning.lite.wrappers import _LiteDataLoader, _LiteModule, _LiteOptimizer +from tests.helpers.runif import RunIf + + +class EmptyLite(LightningLite): + def run(self): + pass + + +def test_lite_module_wraps(): + """Test that the wrapped module is accessible via the property.""" + module = Mock() + assert _LiteModule(module, Mock()).module is module + + +@RunIf(min_gpus=1) +@pytest.mark.parametrize( + "precision, input_type, expected_type", + [ + (32, torch.float16, torch.float32), + (32, torch.float32, torch.float32), + (32, torch.float64, torch.float32), + (16, torch.float32, torch.float16), + (16, torch.float64, torch.float16), + # ("mixed", torch.float32, torch.float16), # TODO: support precision="mixed" + ], +) +def test_lite_module_forward_conversion(precision, input_type, expected_type): + """Test that the LiteModule performs autocasting on the input tensors and during forward().""" + lite = EmptyLite(precision=precision, accelerator="gpu", devices=1) + device = torch.device("cuda", 0) + + def check_autocast(forward_input): + assert precision not in (16, "mixed") or torch.is_autocast_enabled() + return forward_input + + module = Mock(wraps=torch.nn.Linear(1, 1), side_effect=check_autocast) + lite_module = _LiteModule(module, lite._accelerator).to(device) + out = lite_module(torch.rand(1, dtype=input_type, device=device)) + assert module.call_args[0][0].dtype == expected_type + assert out.dtype == torch.get_default_dtype() + + +@pytest.mark.parametrize( + "src_device, dest_device", + [ + (torch.device("cpu"), torch.device("cpu")), + pytest.param(torch.device("cpu"), torch.device("cuda", 0), marks=RunIf(min_gpus=1)), + pytest.param(torch.device("cuda", 0), torch.device("cpu"), marks=RunIf(min_gpus=1)), + ], +) +def test_lite_dataloader_device_placement(src_device, dest_device): + """Test that the LiteDataLoader moves data to the device in its iterator.""" + sample0 = torch.tensor(0, device=src_device) + sample1 = torch.tensor(1, device=src_device) + sample2 = {"data": torch.tensor(2, device=src_device)} + sample3 = {"data": torch.tensor(3, device=src_device)} + data = [sample0, sample1, sample2, sample3] + lite_dataloader = _LiteDataLoader(device=dest_device, dataset=data, batch_size=2) + iterator = iter(lite_dataloader) + + batch0 = next(iterator) + assert torch.equal(batch0, torch.tensor([0, 1], device=dest_device)) + + batch1 = next(iterator) + assert torch.equal(batch1["data"], torch.tensor([2, 3], device=dest_device)) + + +def test_lite_optimizer_wraps(): + """Test that the LiteOptimizer fully wraps the optimizer.""" + optimizer_cls = torch.optim.SGD + optimizer = Mock(spec=optimizer_cls) + lite_optimizer = _LiteOptimizer(optimizer, Mock()) + assert lite_optimizer.optimizer is optimizer + assert isinstance(lite_optimizer, optimizer_cls) + + +def test_lite_optimizer_steps(): + """Test that the LiteOptimizer forwards the step() and zero_grad() calls to the wrapped optimizer.""" + optimizer = Mock() + accelerator = Mock() + lite_optimizer = _LiteOptimizer(optimizer=optimizer, accelerator=accelerator) + lite_optimizer.step() + accelerator.optimizer_step.assert_called_once() + accelerator.optimizer_step.assert_called_with(optimizer, lambda_closure=None, model=None) + lite_optimizer.zero_grad() + optimizer.zero_grad.assert_called_once() From fe65b7477eee5c23c431c2794d379fd80a160d8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 20 Oct 2021 15:39:03 +0200 Subject: [PATCH 148/331] update closure --- pytorch_lightning/accelerators/accelerator.py | 4 ++-- pytorch_lightning/lite/wrappers.py | 6 ++++++ pytorch_lightning/plugins/precision/deepspeed_precision.py | 2 +- pytorch_lightning/plugins/precision/native_amp.py | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/accelerators/accelerator.py b/pytorch_lightning/accelerators/accelerator.py index 7e86f9f700986..5c6d238568e2d 100644 --- a/pytorch_lightning/accelerators/accelerator.py +++ b/pytorch_lightning/accelerators/accelerator.py @@ -317,8 +317,8 @@ def backward(self, closure_loss: Tensor, *args: Any, **kwargs: Any) -> Tensor: def optimizer_step( self, optimizer: Optimizer, - opt_idx: int = 0, - lambda_closure: Optional[Callable] = None, + opt_idx, + lambda_closure: Callable, model: Optional[Union[Module, "pl.LightningModule"]] = None, **kwargs: Any ) -> None: diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 90924be2f23b6..5f9a12147c1eb 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -23,6 +23,10 @@ from pytorch_lightning.utilities.apply_func import apply_to_collection, move_data_to_device +def _do_nothing_closure() -> None: + return None + + class _LiteOptimizer: def __init__(self, optimizer: Optimizer, accelerator: Accelerator) -> None: self.__dict__ = {k: v for k, v in optimizer.__dict__.items() if k not in ("step", "__del__")} @@ -59,8 +63,10 @@ def param_groups(self, param_groups): self._optimizer.param_groups = param_groups def step(self, closure: Optional[Callable] = None) -> None: + closure = closure or _do_nothing_closure self._accelerator.optimizer_step( self._optimizer, + opt_idx=0, lambda_closure=closure, model=None, ) diff --git a/pytorch_lightning/plugins/precision/deepspeed_precision.py b/pytorch_lightning/plugins/precision/deepspeed_precision.py index d6ad215d3e486..ad7265667e763 100644 --- a/pytorch_lightning/plugins/precision/deepspeed_precision.py +++ b/pytorch_lightning/plugins/precision/deepspeed_precision.py @@ -43,7 +43,7 @@ def pre_optimizer_step( **kwargs: Any, ) -> bool: """Hook to do something before each optimizer step.""" - result = lambda_closure() if lambda_closure is not None else None # DeepSpeed does not support closures + result = lambda_closure() # DeepSpeed does not support closures super().pre_optimizer_step(model, optimizer, optimizer_idx, lambda_closure, **kwargs) # in manual optimization, the closure does not return a value if isinstance(model, pl.LightningModule) and model.automatic_optimization and result is None: diff --git a/pytorch_lightning/plugins/precision/native_amp.py b/pytorch_lightning/plugins/precision/native_amp.py index 08b8080715d84..74651ce3e4a32 100644 --- a/pytorch_lightning/plugins/precision/native_amp.py +++ b/pytorch_lightning/plugins/precision/native_amp.py @@ -90,7 +90,7 @@ def pre_optimizer_step( raise MisconfigurationException( f"Native AMP and the LBFGS optimizer are not compatible (optimizer {optimizer_idx})." ) - result = lambda_closure() if lambda_closure is not None else None # native amp does not support closures + result = lambda_closure() # native amp does not support closures self.scaler.unscale_(optimizer) super().pre_optimizer_step(model, optimizer, optimizer_idx, lambda_closure, **kwargs) skipped_backward = result is None From b4a0c4a17067bf9efe5a293d5bbf1f200ec6c2bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 20 Oct 2021 15:40:44 +0200 Subject: [PATCH 149/331] update zero_grad --- pytorch_lightning/lite/wrappers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 5f9a12147c1eb..e2bafd587a2e5 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -71,8 +71,8 @@ def step(self, closure: Optional[Callable] = None) -> None: model=None, ) - def zero_grad(self, set_to_none: bool = False) -> None: - self._optimizer.zero_grad(set_to_none=set_to_none) + def zero_grad(self, *args: Any, **kwargs: Any) -> None: + self._optimizer.zero_grad(*args, **kwargs) class _LiteModule(nn.Module): From e5bd182b5cf806b48030941acb350cfec78e99e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 20 Oct 2021 18:17:58 +0200 Subject: [PATCH 150/331] tests for device --- pytorch_lightning/lite/wrappers.py | 4 ++ tests/lite/test_lite.py | 63 ++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 tests/lite/test_lite.py diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index e2bafd587a2e5..989a4ff7aeb23 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -109,6 +109,10 @@ def __init__(self, device: Optional[torch.device] = None, **dl_kwargs: Any) -> N super().__init__(**dl_kwargs) self._device = device + @property + def device(self) -> Optional[torch.device]: + return self._device + def __iter__(self) -> Union[Iterator[Any], Generator[Any, None, None]]: # type: ignore[override] iterator = super().__iter__() if self._device is None: diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py new file mode 100644 index 0000000000000..ae5880c9ace63 --- /dev/null +++ b/tests/lite/test_lite.py @@ -0,0 +1,63 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from unittest import mock +from unittest.mock import Mock, patch, PropertyMock + +import pytest +import torch +from torch.utils.data import DataLoader + +from pytorch_lightning.lite import LightningLite +from pytorch_lightning.lite.wrappers import _LiteDataLoader + + +class EmptyLite(LightningLite): + def run(self): + pass + + +def test_setup_dataloaders_return_type(): + lite = EmptyLite() + + # single dataloader + lite_dataloader = lite.setup_dataloaders(DataLoader(range(2))) + assert isinstance(lite_dataloader, _LiteDataLoader) + + # multiple dataloaders + dataset0 = range(2) + dataset1 = range(3) + dataloader0 = DataLoader(dataset0) + dataloader1 = DataLoader(dataset1) + lite_dataloader0, lite_dataloader1 = lite.setup_dataloaders(dataloader0, dataloader1) + assert isinstance(lite_dataloader0, _LiteDataLoader) + assert isinstance(lite_dataloader1, _LiteDataLoader) + assert lite_dataloader0.dataset is dataset0 + assert lite_dataloader1.dataset is dataset1 + + +@mock.patch( + "pytorch_lightning.lite.lite.LightningLite.device", + new_callable=PropertyMock, + return_value=torch.device("cuda", 1), +) +def test_setup_dataloaders_move_to_device(lite_device_mock): + lite = EmptyLite() + lite_dataloaders = lite.setup_dataloaders(DataLoader(Mock()), DataLoader(Mock()), move_to_device=False) + assert all(dl.device is None for dl in lite_dataloaders) + lite_device_mock.assert_not_called() + + lite = EmptyLite() + lite_dataloaders = lite.setup_dataloaders(DataLoader(Mock()), DataLoader(Mock()), move_to_device=True) + assert all(dl.device == torch.device("cuda", 1) for dl in lite_dataloaders) + lite_device_mock.assert_called() From 74b11eb1f1e807df8f260e11381501e04b41c810 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 20 Oct 2021 22:56:54 +0200 Subject: [PATCH 151/331] merge conflict fixes --- pytorch_lightning/lite/wrappers.py | 2 +- pytorch_lightning/plugins/training_type/tpu_spawn.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index e2bafd587a2e5..abdce283f8008 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -68,7 +68,7 @@ def step(self, closure: Optional[Callable] = None) -> None: self._optimizer, opt_idx=0, lambda_closure=closure, - model=None, + model=self._accelerator.model, ) def zero_grad(self, *args: Any, **kwargs: Any) -> None: diff --git a/pytorch_lightning/plugins/training_type/tpu_spawn.py b/pytorch_lightning/plugins/training_type/tpu_spawn.py index 9c3cd50491d58..4db8cfc9e308c 100644 --- a/pytorch_lightning/plugins/training_type/tpu_spawn.py +++ b/pytorch_lightning/plugins/training_type/tpu_spawn.py @@ -20,6 +20,7 @@ import torch import torch.multiprocessing as mp +from torch.nn import Module from torch.utils.data import DataLoader import pytorch_lightning as pl @@ -261,9 +262,6 @@ def get_mp_spawn_kwargs(self, trainer: Optional["pl.Trainer"] = None) -> Dict[st "start_method": self.start_method, } - def optimizer_step(self, optimizer: Optimizer, lambda_closure: Callable, **kwargs) -> None: - xm.optimizer_step(optimizer, barrier=False, optimizer_args={"closure": lambda_closure, **kwargs}) - def spawn(self, function: Callable, *args: Any, **kwargs: Any) -> None: xmp.spawn(self._wrapped_function, args=(function, args, kwargs), **self.get_mp_spawn_kwargs()) From 4c81c78e21fe521271468f5a34fcaf255383c386 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 03:53:03 +0200 Subject: [PATCH 152/331] add tests for distributed sampler --- tests/lite/test_lite.py | 49 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index ae5880c9ace63..8c5fcaf0d294e 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -12,14 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. from unittest import mock -from unittest.mock import Mock, patch, PropertyMock +from unittest.mock import Mock, PropertyMock import pytest import torch -from torch.utils.data import DataLoader +from torch.utils.data import DataLoader, Sampler, DistributedSampler from pytorch_lightning.lite import LightningLite from pytorch_lightning.lite.wrappers import _LiteDataLoader +from pytorch_lightning.utilities.exceptions import MisconfigurationException class EmptyLite(LightningLite): @@ -28,6 +29,7 @@ def run(self): def test_setup_dataloaders_return_type(): + """Test that the setup method returns the dataloaders wrapped as LiteDataLoader and in the right order.""" lite = EmptyLite() # single dataloader @@ -35,8 +37,8 @@ def test_setup_dataloaders_return_type(): assert isinstance(lite_dataloader, _LiteDataLoader) # multiple dataloaders - dataset0 = range(2) - dataset1 = range(3) + dataset0 = Mock() + dataset1 = Mock() dataloader0 = DataLoader(dataset0) dataloader1 = DataLoader(dataset1) lite_dataloader0, lite_dataloader1 = lite.setup_dataloaders(dataloader0, dataloader1) @@ -52,6 +54,7 @@ def test_setup_dataloaders_return_type(): return_value=torch.device("cuda", 1), ) def test_setup_dataloaders_move_to_device(lite_device_mock): + """Test that the setup configures LiteDataLoader to move the data to the device automatically.""" lite = EmptyLite() lite_dataloaders = lite.setup_dataloaders(DataLoader(Mock()), DataLoader(Mock()), move_to_device=False) assert all(dl.device is None for dl in lite_dataloaders) @@ -61,3 +64,41 @@ def test_setup_dataloaders_move_to_device(lite_device_mock): lite_dataloaders = lite.setup_dataloaders(DataLoader(Mock()), DataLoader(Mock()), move_to_device=True) assert all(dl.device == torch.device("cuda", 1) for dl in lite_dataloaders) lite_device_mock.assert_called() + + +def test_setup_dataloaders_distributed_sampler_not_needed(): + """Test that replace_sampler option has no effect when no distributed sampler is needed.""" + custom_sampler = Mock(spec=Sampler) + dataloader = DataLoader(Mock(), sampler=custom_sampler) + + # keep the custom sampler when not needed to replace + lite = EmptyLite() + lite_dataloader = lite.setup_dataloaders(dataloader, replace_sampler=True) + assert lite_dataloader.sampler is custom_sampler + + +@pytest.mark.parametrize("strategy", LightningLite._supported_strategy_types()) +def test_setup_dataloaders_replace_custom_sampler(strategy): + """Test that asking to replace a custom sampler results in an error when a distributed sampler would be needed.""" + custom_sampler = Mock(spec=Sampler) + dataloader = DataLoader(Mock(), sampler=custom_sampler) + + # explicitly asking to replace when a custom sampler is already configured raises an exception + lite = EmptyLite(accelerator="cpu", strategy=strategy, devices=2) + if getattr(lite._strategy, "is_distributed", False): + with pytest.raises(MisconfigurationException, match="You seem to have configured a sampler in your DataLoader"): + lite.setup_dataloaders(dataloader, replace_sampler=True) + + # setting `replace_sampler=False` leaves the sampler untouched + lite_dataloader = lite.setup_dataloaders(dataloader, replace_sampler=False) + assert lite_dataloader.sampler is custom_sampler + + +@pytest.mark.parametrize("strategy", LightningLite._supported_strategy_types()) +@pytest.mark.parametrize("shuffle", [True, False]) +def test_setup_dataloaders_replace_standard_sampler(shuffle, strategy): + """Test that Lite replaces the default samplers with DistributedSampler automatically.""" + lite = EmptyLite(accelerator="cpu", strategy=strategy, devices=2) + is_distributed = getattr(lite._strategy, "is_distributed", False) + lite_dataloader = lite.setup_dataloaders(DataLoader(range(3), shuffle=shuffle)) + assert not is_distributed or isinstance(lite_dataloader.sampler, DistributedSampler) From b33dda2dae70beed87f5d60e11ccec6fa2316c36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 03:57:33 +0200 Subject: [PATCH 153/331] update is_distrib access --- tests/lite/test_lite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 8c5fcaf0d294e..e0be0efd7c546 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -85,7 +85,7 @@ def test_setup_dataloaders_replace_custom_sampler(strategy): # explicitly asking to replace when a custom sampler is already configured raises an exception lite = EmptyLite(accelerator="cpu", strategy=strategy, devices=2) - if getattr(lite._strategy, "is_distributed", False): + if lite._accelerator_connector.is_distributed: with pytest.raises(MisconfigurationException, match="You seem to have configured a sampler in your DataLoader"): lite.setup_dataloaders(dataloader, replace_sampler=True) @@ -99,6 +99,6 @@ def test_setup_dataloaders_replace_custom_sampler(strategy): def test_setup_dataloaders_replace_standard_sampler(shuffle, strategy): """Test that Lite replaces the default samplers with DistributedSampler automatically.""" lite = EmptyLite(accelerator="cpu", strategy=strategy, devices=2) - is_distributed = getattr(lite._strategy, "is_distributed", False) + is_distributed = lite._accelerator_connector.is_distributed lite_dataloader = lite.setup_dataloaders(DataLoader(range(3), shuffle=shuffle)) assert not is_distributed or isinstance(lite_dataloader.sampler, DistributedSampler) From 5935ce47d3988a7a0ac65dab29a43c4466efc83d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 10:53:52 +0200 Subject: [PATCH 154/331] remove comment --- pytorch_lightning/plugins/training_type/deepspeed.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pytorch_lightning/plugins/training_type/deepspeed.py b/pytorch_lightning/plugins/training_type/deepspeed.py index 13f454e44c7f4..74c6b07fbfb3e 100644 --- a/pytorch_lightning/plugins/training_type/deepspeed.py +++ b/pytorch_lightning/plugins/training_type/deepspeed.py @@ -343,7 +343,6 @@ def _load_config(self, config): config = json.load(f) return config - # getting called by Lightning trainer AND Lite def setup_distributed(self): reset_seed() From 12de35ce92d4553a5c8a0a2bc4f4f0f065b2d6fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 11:04:50 +0200 Subject: [PATCH 155/331] update spawn() for tpu_spawn plugin --- pytorch_lightning/plugins/training_type/ddp_spawn.py | 5 ++++- pytorch_lightning/plugins/training_type/tpu_spawn.py | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pytorch_lightning/plugins/training_type/ddp_spawn.py b/pytorch_lightning/plugins/training_type/ddp_spawn.py index 1c9a654bc0b9e..0f0008254d308 100644 --- a/pytorch_lightning/plugins/training_type/ddp_spawn.py +++ b/pytorch_lightning/plugins/training_type/ddp_spawn.py @@ -183,6 +183,9 @@ def spawn(self, function: Callable, *args: Any, **kwargs: Any) -> Any: These arguments must be pickleable. **kwargs: Optional named arguments that will be passed to the function in addition to the process index. These arguments must be pickleable. + + Return: + The output of the function of process 0. """ os.environ["MASTER_PORT"] = str(self.cluster_environment.master_port()) smp = mp.get_context("spawn") @@ -195,7 +198,7 @@ def _wrapped_function( ) -> None: self._worker_setup(process_idx) result = function(*args, **kwargs) - if self.is_global_zero: + if self.local_rank == 0: return_queue.put(move_data_to_device(result, "cpu")) def _worker_setup(self, process_idx: int): diff --git a/pytorch_lightning/plugins/training_type/tpu_spawn.py b/pytorch_lightning/plugins/training_type/tpu_spawn.py index 6f8a2bf648523..13467cedf9fa2 100644 --- a/pytorch_lightning/plugins/training_type/tpu_spawn.py +++ b/pytorch_lightning/plugins/training_type/tpu_spawn.py @@ -260,8 +260,11 @@ def get_mp_spawn_kwargs(self, trainer: Optional["pl.Trainer"] = None) -> Dict[st "start_method": self.start_method, } - def spawn(self, function: Callable, *args: Any, **kwargs: Any) -> None: - xmp.spawn(self._wrapped_function, args=(function, args, kwargs), **self.get_mp_spawn_kwargs()) + def spawn(self, function: Callable, *args: Any, **kwargs: Any) -> Any: + smp = mp.get_context(self.start_method or "fork") + return_queue = smp.SimpleQueue() + xmp.spawn(self._wrapped_function, args=(function, args, kwargs, return_queue), **self.get_mp_spawn_kwargs()) + return return_queue.get() def _worker_setup(self, process_idx: int): reset_seed() From 56271bc7bb7326f76226ad514604b2187b61dcb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 11:10:35 +0200 Subject: [PATCH 156/331] update bloat check --- tests/lite/test_parity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/lite/test_parity.py b/tests/lite/test_parity.py index 5efd9bae63914..423a606561d2d 100644 --- a/tests/lite/test_parity.py +++ b/tests/lite/test_parity.py @@ -34,7 +34,7 @@ from pytorch_lightning.plugins.training_type.ddp_spawn import DDPSpawnPlugin from pytorch_lightning.utilities.apply_func import apply_to_collection, move_data_to_device from pytorch_lightning.utilities.cloud_io import atomic_save -from pytorch_lightning.utilities.imports import _TORCH_BFLOAT_AVAILABLE +from pytorch_lightning.utilities.imports import _TORCH_GREATER_EQUAL_DEV_1_10 from tests.helpers.boring_model import RandomDataset from tests.helpers.runif import RunIf @@ -123,7 +123,7 @@ def precision_context(precision, accelerator) -> Generator[None, None, None]: 1, "gpu", marks=pytest.mark.skipif( - not (_TORCH_BFLOAT_AVAILABLE and is_available()), + not (_TORCH_GREATER_EQUAL_DEV_1_10 and is_available()), reason="bfloat16 and requires GPU isn't available.", ), ), From 7b83347875c60de3ccf816f58d74ce10a616cd18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 11:28:15 +0200 Subject: [PATCH 157/331] update optimizer step test --- tests/lite/test_wrappers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/lite/test_wrappers.py b/tests/lite/test_wrappers.py index c58a750afaf03..37c85d05558b2 100644 --- a/tests/lite/test_wrappers.py +++ b/tests/lite/test_wrappers.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from unittest.mock import Mock +from unittest.mock import Mock, ANY import pytest import torch @@ -101,6 +101,6 @@ def test_lite_optimizer_steps(): lite_optimizer = _LiteOptimizer(optimizer=optimizer, accelerator=accelerator) lite_optimizer.step() accelerator.optimizer_step.assert_called_once() - accelerator.optimizer_step.assert_called_with(optimizer, lambda_closure=None, model=None) + accelerator.optimizer_step.assert_called_with(optimizer, opt_idx=0, lambda_closure=ANY, model=accelerator.model) lite_optimizer.zero_grad() optimizer.zero_grad.assert_called_once() From 4f0d82a82ff29b40f888e685ac6657657d3ec68a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Oct 2021 09:30:28 +0000 Subject: [PATCH 158/331] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/lite/test_lite.py | 5 +++-- tests/lite/test_wrappers.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index e0be0efd7c546..0428fb3036f6e 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -16,7 +16,7 @@ import pytest import torch -from torch.utils.data import DataLoader, Sampler, DistributedSampler +from torch.utils.data import DataLoader, DistributedSampler, Sampler from pytorch_lightning.lite import LightningLite from pytorch_lightning.lite.wrappers import _LiteDataLoader @@ -79,7 +79,8 @@ def test_setup_dataloaders_distributed_sampler_not_needed(): @pytest.mark.parametrize("strategy", LightningLite._supported_strategy_types()) def test_setup_dataloaders_replace_custom_sampler(strategy): - """Test that asking to replace a custom sampler results in an error when a distributed sampler would be needed.""" + """Test that asking to replace a custom sampler results in an error when a distributed sampler would be + needed.""" custom_sampler = Mock(spec=Sampler) dataloader = DataLoader(Mock(), sampler=custom_sampler) diff --git a/tests/lite/test_wrappers.py b/tests/lite/test_wrappers.py index 37c85d05558b2..faed290b75629 100644 --- a/tests/lite/test_wrappers.py +++ b/tests/lite/test_wrappers.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from unittest.mock import Mock, ANY +from unittest.mock import ANY, Mock import pytest import torch From f86fd6fd401a292b424d3c53025f8be8bb750286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 11:33:19 +0200 Subject: [PATCH 159/331] add guards to example --- .../pytorch_2_lite_2_lightning.py | 48 ++++++++++--------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/pl_examples/lite_examples/pytorch_2_lite_2_lightning.py b/pl_examples/lite_examples/pytorch_2_lite_2_lightning.py index 511031697dc8a..1c81907816597 100644 --- a/pl_examples/lite_examples/pytorch_2_lite_2_lightning.py +++ b/pl_examples/lite_examples/pytorch_2_lite_2_lightning.py @@ -108,9 +108,10 @@ def main(model: nn.Module, train_dataloader: DataLoader, val_dataloader: DataLoa # 6 / 6: Run the pure PyTorch Loop and train / validate the model. -seed_everything(42) -model = BoringModel() -pure_model_weights = main(model, train_dataloader(), val_dataloader()) +if __name__ == "__main__": + seed_everything(42) + model = BoringModel() + pure_model_weights = main(model, train_dataloader(), val_dataloader()) ############################################################################################# @@ -172,17 +173,18 @@ def run(self, model: nn.Module, train_dataloader: DataLoader, val_dataloader: Da ################################################################################ -seed_everything(42) -lite_model = BoringModel() -lite = LiteTrainer() -lite.run(lite_model, train_dataloader(), val_dataloader()) +if __name__ == "__main__": + seed_everything(42) + lite_model = BoringModel() + lite = LiteTrainer() + lite.run(lite_model, train_dataloader(), val_dataloader()) -############################################################################################# -# Assert the weights are the same # -############################################################################################# + ############################################################################################# + # Assert the weights are the same # + ############################################################################################# -for pure_w, lite_w in zip(pure_model_weights.values(), lite_model.state_dict().values()): - torch.equal(pure_w, lite_w) + for pure_w, lite_w in zip(pure_model_weights.values(), lite_model.state_dict().values()): + torch.equal(pure_w, lite_w) ############################################################################################# @@ -231,16 +233,16 @@ def val_dataloader(self): return val_dataloader() -seed_everything(42) -lightning_module = LightningBoringModel() -datamodule = BoringDataModule() -trainer = Trainer(max_epochs=10) -trainer.fit(lightning_module, datamodule) - +if __name__ == "__main__": + seed_everything(42) + lightning_module = LightningBoringModel() + datamodule = BoringDataModule() + trainer = Trainer(max_epochs=10) + trainer.fit(lightning_module, datamodule) -############################################################################################# -# Assert the weights are the same # -############################################################################################# + ############################################################################################# + # Assert the weights are the same # + ############################################################################################# -for pure_w, lite_w in zip(pure_model_weights.values(), lightning_module.state_dict().values()): - torch.equal(pure_w, lite_w) + for pure_w, lite_w in zip(pure_model_weights.values(), lightning_module.state_dict().values()): + torch.equal(pure_w, lite_w) From f703112b5b57c69e113174a2495ff453b8f08577 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 11:35:52 +0200 Subject: [PATCH 160/331] move scrips to debug folder for removal later on --- pl_examples/{lite_examples/gan => debug}/__init__.py | 0 pl_examples/{lite_examples/simple => debug/gan}/__init__.py | 0 pl_examples/{lite_examples => debug}/gan/gan_example.py | 4 +--- pl_examples/{lite_examples => debug}/gan/models.py | 0 pl_examples/{lite_examples => debug}/gan/run_examples.py | 2 +- pl_examples/debug/simple/__init__.py | 0 pl_examples/{lite_examples => debug}/simple/mnist_example.py | 0 7 files changed, 2 insertions(+), 4 deletions(-) rename pl_examples/{lite_examples/gan => debug}/__init__.py (100%) rename pl_examples/{lite_examples/simple => debug/gan}/__init__.py (100%) rename pl_examples/{lite_examples => debug}/gan/gan_example.py (97%) rename pl_examples/{lite_examples => debug}/gan/models.py (100%) rename pl_examples/{lite_examples => debug}/gan/run_examples.py (88%) create mode 100644 pl_examples/debug/simple/__init__.py rename pl_examples/{lite_examples => debug}/simple/mnist_example.py (100%) diff --git a/pl_examples/lite_examples/gan/__init__.py b/pl_examples/debug/__init__.py similarity index 100% rename from pl_examples/lite_examples/gan/__init__.py rename to pl_examples/debug/__init__.py diff --git a/pl_examples/lite_examples/simple/__init__.py b/pl_examples/debug/gan/__init__.py similarity index 100% rename from pl_examples/lite_examples/simple/__init__.py rename to pl_examples/debug/gan/__init__.py diff --git a/pl_examples/lite_examples/gan/gan_example.py b/pl_examples/debug/gan/gan_example.py similarity index 97% rename from pl_examples/lite_examples/gan/gan_example.py rename to pl_examples/debug/gan/gan_example.py index 9cf4a91ed3ff3..2232ce339f584 100644 --- a/pl_examples/lite_examples/gan/gan_example.py +++ b/pl_examples/debug/gan/gan_example.py @@ -9,7 +9,6 @@ import argparse import os -import random import torch import torch.nn as nn @@ -19,9 +18,8 @@ import torchvision.datasets as dset import torchvision.transforms as transforms import torchvision.utils as vutils -from torch.utils.data import DistributedSampler -from pl_examples.lite_examples.gan.models import Discriminator, Generator, weights_init +from pl_examples.debug.gan.models import Discriminator, Generator, weights_init from pytorch_lightning import seed_everything from pytorch_lightning.lite import LightningLite from pytorch_lightning.lite.wrappers import _LiteModule, _LiteOptimizer diff --git a/pl_examples/lite_examples/gan/models.py b/pl_examples/debug/gan/models.py similarity index 100% rename from pl_examples/lite_examples/gan/models.py rename to pl_examples/debug/gan/models.py diff --git a/pl_examples/lite_examples/gan/run_examples.py b/pl_examples/debug/gan/run_examples.py similarity index 88% rename from pl_examples/lite_examples/gan/run_examples.py rename to pl_examples/debug/gan/run_examples.py index 4133fcd3c6338..3cd724a0639e2 100644 --- a/pl_examples/lite_examples/gan/run_examples.py +++ b/pl_examples/debug/gan/run_examples.py @@ -1,6 +1,6 @@ import argparse -from pl_examples.lite_examples.gan.gan_example import GANTrainer +from pl_examples.debug.gan.gan_example import GANTrainer if __name__ == "__main__": parser = argparse.ArgumentParser() diff --git a/pl_examples/debug/simple/__init__.py b/pl_examples/debug/simple/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/pl_examples/lite_examples/simple/mnist_example.py b/pl_examples/debug/simple/mnist_example.py similarity index 100% rename from pl_examples/lite_examples/simple/mnist_example.py rename to pl_examples/debug/simple/mnist_example.py From 3421372842a9bdc8d38bc35e1bb5b76800c97abc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 14:21:57 +0200 Subject: [PATCH 161/331] test invalid choices --- tests/lite/test_lite.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index e0be0efd7c546..34bb925b887ca 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -18,8 +18,10 @@ import torch from torch.utils.data import DataLoader, Sampler, DistributedSampler +from pytorch_lightning.accelerators import Accelerator from pytorch_lightning.lite import LightningLite from pytorch_lightning.lite.wrappers import _LiteDataLoader +from pytorch_lightning.plugins import TrainingTypePlugin from pytorch_lightning.utilities.exceptions import MisconfigurationException @@ -28,6 +30,18 @@ def run(self): pass +@pytest.mark.parametrize("accelerator", ["coconut", Mock(spec=Accelerator)]) +def test_unsupported_accelerator(accelerator): + with pytest.raises(MisconfigurationException, match=f"`accelerator={repr(accelerator)}` is not a valid choice"): + EmptyLite(accelerator=accelerator) + + +@pytest.mark.parametrize("strategy", ["coconut", Mock(spec=TrainingTypePlugin)]) +def test_unsupported_strategy(strategy): + with pytest.raises(MisconfigurationException, match=f"`strategy={repr(strategy)}` is not a valid choice"): + EmptyLite(strategy=strategy) + + def test_setup_dataloaders_return_type(): """Test that the setup method returns the dataloaders wrapped as LiteDataLoader and in the right order.""" lite = EmptyLite() From 3744ea4aee3f2ff995b60d4e0d662c1d1c57c742 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 14:36:13 +0200 Subject: [PATCH 162/331] test to_device --- tests/lite/test_lite.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 34bb925b887ca..3691a9bd97526 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -23,6 +23,7 @@ from pytorch_lightning.lite.wrappers import _LiteDataLoader from pytorch_lightning.plugins import TrainingTypePlugin from pytorch_lightning.utilities.exceptions import MisconfigurationException +from tests.helpers.runif import RunIf class EmptyLite(LightningLite): @@ -116,3 +117,27 @@ def test_setup_dataloaders_replace_standard_sampler(shuffle, strategy): is_distributed = lite._accelerator_connector.is_distributed lite_dataloader = lite.setup_dataloaders(DataLoader(range(3), shuffle=shuffle)) assert not is_distributed or isinstance(lite_dataloader.sampler, DistributedSampler) + + +@pytest.mark.parametrize( + "accelerator, expected", + [ + ("cpu", torch.device("cpu")), + pytest.param("gpu", torch.device("cuda", 0), marks=RunIf(min_gpus=1)), + pytest.param("tpu", torch.device("xla", 0), marks=RunIf(tpu=True)), + ], +) +def test_to_device(accelerator, expected): + lite = EmptyLite(accelerator=accelerator, devices=1) + + module = torch.nn.Linear(2, 3) + module = lite.to_device(module) + assert all(param.device == expected for param in module.parameters()) + + tensor = torch.rand(2, 2) + tensor = lite.to_device(tensor) + assert tensor.device == expected + + collection = {"data": torch.rand(2, 2)} + collection = lite.to_device(collection) + assert collection["data"].device == expected From d197b495aa4f733ea71297477f444f4713481dce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 14:39:54 +0200 Subject: [PATCH 163/331] save checkpoint --- pytorch_lightning/lite/lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index b16ee73dba769..f43ddb04a76ca 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -308,7 +308,7 @@ def reduce_decision(self, decision: bool) -> bool: return self._strategy.reduce_boolean_decision(decision) def save_checkpoint(self, filepath: Union[str, Path], content: Dict[str, Any]) -> None: - raise NotImplementedError() + self._strategy.save_checkpoint(content, filepath) def execute_on_rank(self, func: Callable, rank: int, *args: Any, **kwargs: Any) -> None: if self.global_rank == rank: From 255187722010ff1189ae4bc74b66d0444e789953 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 14:46:13 +0200 Subject: [PATCH 164/331] update test description --- tests/lite/test_lite.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 3691a9bd97526..58dcf5355a13a 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -128,16 +128,20 @@ def test_setup_dataloaders_replace_standard_sampler(shuffle, strategy): ], ) def test_to_device(accelerator, expected): + """Test that the to_device method can move various objects to the device determined by the accelerator.""" lite = EmptyLite(accelerator=accelerator, devices=1) + # module module = torch.nn.Linear(2, 3) module = lite.to_device(module) assert all(param.device == expected for param in module.parameters()) + # tensor tensor = torch.rand(2, 2) tensor = lite.to_device(tensor) assert tensor.device == expected - collection = {"data": torch.rand(2, 2)} + # collection + collection = {"data": torch.rand(2, 2), "int": 1} collection = lite.to_device(collection) assert collection["data"].device == expected From 5c46acda93eb65d6a0d7e2d298cb6bc76ef6fb01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 14:56:53 +0200 Subject: [PATCH 165/331] document public api --- pytorch_lightning/lite/lite.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index f43ddb04a76ca..cfffb746b04c2 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -305,12 +305,42 @@ def barrier(self) -> None: self._strategy.barrier() def reduce_decision(self, decision: bool) -> bool: + """Reduce a boolean decision across processes. + + Use this for example to determine an early stopping condition, in which case you want to stop if any of + the processes determine to stop. + + Args: + decision: The decision on the current process + + Return: + If at least one of the processes enters with ``decision=True``, then all processes will return `True`. + Otherwise returns ``False``. + """ return self._strategy.reduce_boolean_decision(decision) def save_checkpoint(self, filepath: Union[str, Path], content: Dict[str, Any]) -> None: + """Save a checkpoint contents to a file. + + How and which processes save gets determined by the `strategy`. For example, the `ddp` strategy + saves checkpoints only on process 0. + + Args: + filepath: A path to where the file should be saved + content: A dictionary with contents, i.e., the state dict of your model + """ self._strategy.save_checkpoint(content, filepath) def execute_on_rank(self, func: Callable, rank: int, *args: Any, **kwargs: Any) -> None: + """Execute the given function only on the given process. + + Args: + func: The function to execute + rank: The index of the process across all devices and nodes (global rank). This value must be an integer + in the range ``[0, self.world_size - 1]``. + *args: Optional positional arguments passed to the function + **kwargs: Optional named arguments passed to the function + """ if self.global_rank == rank: func(*args, **kwargs) From 98c80668d6177fd17214bd6048bfc56589b14876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 15:11:33 +0200 Subject: [PATCH 166/331] add api docs for wrappers --- pytorch_lightning/lite/wrappers.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index f144067a8c6d6..1639617a97177 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -29,6 +29,15 @@ def _do_nothing_closure() -> None: class _LiteOptimizer: def __init__(self, optimizer: Optimizer, accelerator: Accelerator) -> None: + """LiteOptimizer is a thin wrapper around the :class:`~torch.optim.Optimizer` that delegates the optimizer + step calls to the accelerator/strategy plugin. + + The underlying wrapped optimizer object can be accessed via the property :attr:`optimizer`. + + Args: + optimizer: The optimizer to wrap + accelerator: Reference to the accelerator for handling the optimizer step + """ self.__dict__ = {k: v for k, v in optimizer.__dict__.items() if k not in ("step", "__del__")} self.__class__ = type("Lite" + optimizer.__class__.__name__, (self.__class__, optimizer.__class__), {}) self._optimizer = optimizer @@ -76,7 +85,17 @@ def zero_grad(self, *args: Any, **kwargs: Any) -> None: class _LiteModule(nn.Module): + # TODO: Pass in the precision plugin instead of accelerator def __init__(self, module: nn.Module, accelerator: Accelerator) -> None: + """The LiteModule is a thin wrapper around the :class:`torch.nn.Module` and handles precision / autocast + automatically for the forward pass. + + The underlying wrapped module can be accessed via the property :attr:`module`. + + Args: + module: The module to wrap + accelerator: Reference to the accelerator for handling precision context + """ super().__init__() self._module = module self._accelerator = accelerator @@ -86,6 +105,7 @@ def module(self) -> nn.Module: return self._module def forward(self, *args: Any, **kwargs: Any) -> Any: + """Casts all inputs to the right precision and handles autocast for operations in the module forward method.""" precision = self._accelerator.precision_plugin.precision precision_to_type = { "mixed": torch.float16, @@ -106,6 +126,14 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: class _LiteDataLoader(DataLoader): def __init__(self, device: Optional[torch.device] = None, **dl_kwargs: Any) -> None: + """The LiteDataLoader is an extension of the PyTorch :class:`~torch.utils.data.DataLoader` that adds additional + features such as moving the data to the device automatically. + + Args: + device: The device to which the data should be moved. By default the device is `None` and no data + transfers will be made (identical behavior as :class:`~torch.utils.data.DataLoader`). + **dl_kwargs: Accepts all arguments that the PyTorch :class:`~torch.utils.data.DataLoader` accepts. + """ super().__init__(**dl_kwargs) self._device = device From 079fd279b34c98ae34ee8356b8e3a51e48e6236e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Oct 2021 13:13:32 +0000 Subject: [PATCH 167/331] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytorch_lightning/lite/wrappers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 1639617a97177..9ab8703d2b529 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -105,7 +105,8 @@ def module(self) -> nn.Module: return self._module def forward(self, *args: Any, **kwargs: Any) -> Any: - """Casts all inputs to the right precision and handles autocast for operations in the module forward method.""" + """Casts all inputs to the right precision and handles autocast for operations in the module forward + method.""" precision = self._accelerator.precision_plugin.precision precision_to_type = { "mixed": torch.float16, @@ -126,8 +127,8 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: class _LiteDataLoader(DataLoader): def __init__(self, device: Optional[torch.device] = None, **dl_kwargs: Any) -> None: - """The LiteDataLoader is an extension of the PyTorch :class:`~torch.utils.data.DataLoader` that adds additional - features such as moving the data to the device automatically. + """The LiteDataLoader is an extension of the PyTorch :class:`~torch.utils.data.DataLoader` that adds + additional features such as moving the data to the device automatically. Args: device: The device to which the data should be moved. By default the device is `None` and no data From a4e035c73d99f0772c200adce101e48fd51a6bd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 16:37:52 +0200 Subject: [PATCH 168/331] simple tests --- tests/lite/test_lite.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 6cd26b75c6d7e..0e14900796d60 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -21,7 +21,7 @@ from pytorch_lightning.accelerators import Accelerator from pytorch_lightning.lite import LightningLite from pytorch_lightning.lite.wrappers import _LiteDataLoader -from pytorch_lightning.plugins import TrainingTypePlugin +from pytorch_lightning.plugins import TrainingTypePlugin, PrecisionPlugin from pytorch_lightning.utilities.exceptions import MisconfigurationException from tests.helpers.runif import RunIf @@ -146,3 +146,26 @@ def test_to_device(accelerator, expected): collection = {"data": torch.rand(2, 2), "int": 1} collection = lite.to_device(collection) assert collection["data"].device == expected + + +def test_rank_properties(): + """Test that the rank properties are determined by the strategy.""" + lite = EmptyLite() + lite._strategy = Mock(spec=TrainingTypePlugin) + lite._strategy.world_size = 1000 + assert lite.world_size == 1000 + lite._strategy.global_rank = 100 + assert lite.global_rank == 100 + lite._strategy.local_rank = 10 + assert lite.local_rank == 10 + lite._strategy.node_rank = 1 + assert lite.node_rank == 1 + + +def test_backward(): + """Test that backward() calls into the precision plugin.""" + lite = EmptyLite() + lite._precision_plugin = Mock(spec=PrecisionPlugin) + loss = Mock() + lite.backward(loss, "arg", keyword="kwarg") + lite._precision_plugin._run_backward.assert_called_with(loss, None, "arg", keyword="kwarg") From b14f22ec0b1208a939b4e7c1e8147fd8275eecfb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Oct 2021 14:39:21 +0000 Subject: [PATCH 169/331] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/lite/test_lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 0e14900796d60..8315b08a7066f 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -21,7 +21,7 @@ from pytorch_lightning.accelerators import Accelerator from pytorch_lightning.lite import LightningLite from pytorch_lightning.lite.wrappers import _LiteDataLoader -from pytorch_lightning.plugins import TrainingTypePlugin, PrecisionPlugin +from pytorch_lightning.plugins import PrecisionPlugin, TrainingTypePlugin from pytorch_lightning.utilities.exceptions import MisconfigurationException from tests.helpers.runif import RunIf From eadc10d75eaa50e5cc7f227f4609d75b9fdb9814 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Thu, 21 Oct 2021 15:50:04 +0100 Subject: [PATCH 170/331] Add more LightningLite tests (#10047) --- pytorch_lightning/lite/lite.py | 80 +++++++- .../plugins/precision/deepspeed_precision.py | 6 +- tests/lite/test_lite_api.py | 191 ++++++++++++++++++ 3 files changed, 271 insertions(+), 6 deletions(-) create mode 100644 tests/lite/test_lite_api.py diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index cfffb746b04c2..3c094f3924961 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -39,6 +39,7 @@ from pytorch_lightning.trainer.connectors.accelerator_connector import AcceleratorConnector from pytorch_lightning.trainer.data_loading import TrainerDataLoadingMixin from pytorch_lightning.utilities import DeviceType, DistributedType, move_data_to_device +from pytorch_lightning.utilities.apply_func import apply_to_collection, convert_to_tensors from pytorch_lightning.utilities.data import has_iterable_dataset from pytorch_lightning.utilities.exceptions import MisconfigurationException @@ -106,6 +107,8 @@ def __init__( # wrap the run method so we can inject setup logic or spawn processes for the user setattr(self, "run", self._run_wrapper(self.run)) + self._number_of_models: int = 0 + @property def device(self) -> torch.device: """The current device this process runs on. @@ -134,6 +137,15 @@ def world_size(self) -> int: """The total number of processes running across all devices and nodes.""" return getattr(self._strategy, "world_size", 1) + @property + def is_global_zero(self) -> bool: + """Wether this rank is rank zero.""" + return self._strategy.is_global_zero + + @property + def _is_using_multiple_models(self) -> bool: + return self._number_of_models > 1 + @abstractmethod def run(self, *args: Any, **kwargs: Any) -> Any: """All the code inside this run method gets accelerated by Lite. @@ -148,7 +160,7 @@ def setup( model: nn.Module, optimizers: Union[Optimizer, List[Optimizer]], move_to_device: bool = True, - ) -> Tuple[nn.Module, Union[_LiteOptimizer, List[_LiteOptimizer]]]: + ) -> Tuple[_LiteModule, Union[_LiteOptimizer, List[_LiteOptimizer]]]: """Setup a model and its optimizers for accelerated training. Args: @@ -163,6 +175,8 @@ def setup( # wrap all objects passed in and return them in the same order optimizers = [optimizers] if isinstance(optimizers, Optimizer) else optimizers + self._validate_setup(model, optimizers) + if move_to_device: params_on_cpu = dict(model.named_parameters()) model = self.to_device(model) @@ -178,6 +192,7 @@ def setup( model, optimizers = self._setup_model_and_optimizers(model, optimizers) optimizers = optimizers[0] if len(optimizers) == 1 else optimizers + self._number_of_models += 1 return model, optimizers def setup_dataloaders( @@ -197,6 +212,7 @@ def setup_dataloaders( Returns: The wrapped dataloaders, in the same order they were passed in. """ + self._validate_setup_dataloaders(*dataloaders) # user can call this method independently instead of the general purpose setup method dataloaders = [ self._setup_dataloader(dataloader, replace_sampler=replace_sampler, move_to_device=move_to_device) @@ -240,7 +256,7 @@ def _setup_dataloader( dataloader = _LiteDataLoader(device=device, **kwargs) return self._strategy.process_dataloader(dataloader) - def backward(self, tensor: Tensor, *args: Any, **kwargs: Any) -> None: + def backward(self, tensor: Tensor, *args: Any, model: Optional[_LiteModule] = None, **kwargs: Any) -> None: """Replaces ``loss.backward()`` in your training loop. Handles precision and automatically for you. Args: @@ -248,6 +264,15 @@ def backward(self, tensor: Tensor, *args: Any, **kwargs: Any) -> None: *args: Optional positional arguments passed to the underlying backward function. **kwargs: Optional named keyword arguments passed to the underlying backward function. """ + if self._is_using_multiple_models and isinstance(self._strategy, DeepSpeedPlugin): + if not isinstance(model, _LiteModule): + raise MisconfigurationException( + "When using multiple models + deepspeed, please provide the model used to perform the optimization." + ) + + # requires to attach the current deepSpeed engine for the `optimizer.step` call. + self._strategy.model = model._module + self._precision_plugin._run_backward(tensor, self._strategy.model, *args, **kwargs) @contextmanager @@ -319,6 +344,28 @@ def reduce_decision(self, decision: bool) -> bool: """ return self._strategy.reduce_boolean_decision(decision) + def all_gather( + self, data: Union[torch.Tensor, Dict, List, Tuple], group: Optional[Any] = None, sync_grads: bool = False + ): + r""" + Gather tensors or collections of tensors from multiple processes. + + Args: + data: int, float, tensor of shape (batch, ...), or a (possibly nested) collection thereof. + group: the process group to gather results from. Defaults to all processes (world) + sync_grads: flag that allows users to synchronize gradients for the all_gather operation + + Return: + A tensor of shape (world_size, batch, ...), or if the input was a collection + the output will also be a collection with tensors of this shape. + """ + group = group if group is not None else torch.distributed.group.WORLD + data = convert_to_tensors(data, device=self.device) + return apply_to_collection(data, torch.Tensor, self._strategy.all_gather, group=group, sync_grads=sync_grads) + + def broadcast(self, obj: object, src: int = 0) -> object: + return self._strategy.broadcast(obj, src=src) + def save_checkpoint(self, filepath: Union[str, Path], content: Dict[str, Any]) -> None: """Save a checkpoint contents to a file. @@ -350,9 +397,17 @@ def _run_wrapper(self, run_method: Callable) -> Callable: def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> Any: self._set_plugin_specific_precision_variables() self._accelerator.setup_environment() + + run_fn = partial(self._run_method_wrapper, run_method, *args, **kwargs) + if isinstance(self._strategy, DDPSpawnPlugin): - return self._strategy.spawn(run_method, *args, **kwargs) + return self._strategy.spawn(run_fn) else: + return run_fn() + + def _run_method_wrapper(self, run_method: Callable, *args: Any, **kwargs: Any) -> Any: + # requires to apply sharded context to prevent OOM + with self._strategy.model_sharded_context(): return run_method(*args, **kwargs) def _set_plugin_specific_precision_variables(self) -> None: @@ -375,7 +430,7 @@ def _setup_model_and_optimizers( ) -> Tuple[_LiteModule, List[_LiteOptimizer]]: # Let accelerator/plugin wrap and connect the models and optimizers [model], optimizers = self._strategy._setup_models_and_optimizers([model], optimizers) - model = _LiteModule(module=model, accelerator=self._accelerator) + model = _LiteModule(model, self._accelerator) optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self._accelerator) for optimizer in optimizers] return model, optimizers @@ -405,7 +460,7 @@ def _check_strategy_support(self, strategy: Optional[Union[str, TrainingTypePlug if strategy is None: return supported = [t.lower() for t in self._supported_strategy_types()] - if not isinstance(strategy, (TrainingTypePlugin, str)) or strategy not in supported: + if not isinstance(strategy, (TrainingTypePlugin, str)) and strategy not in supported: raise MisconfigurationException( f"`strategy={repr(strategy)}` is not a valid choice." f" Choose one of {supported} or pass in a `TrainingTypePlugin` instance." @@ -431,3 +486,18 @@ def _supported_strategy_types() -> Sequence[str]: DistributedType.DDP_SHARDED, DistributedType.DDP_SHARDED_SPAWN, ) + + @staticmethod + def _validate_setup(model: nn.Module, optimizers: List[Optimizer]) -> None: + if isinstance(model, _LiteModule): + raise MisconfigurationException("A module should be passed only once to the ``setup`` method") + + if any(isinstance(opt, _LiteOptimizer) for opt in optimizers): + raise MisconfigurationException("An optimizer should be passed only once to the ``setup`` method") + + @staticmethod + def _validate_setup_dataloaders(*dataloaders: Union[DataLoader, List[DataLoader]]) -> None: + if any(isinstance(dl, _LiteDataLoader) for dl in dataloaders): + raise MisconfigurationException( + "A dataloader should be passed only once to the ``setup_dataloaders`` method" + ) diff --git a/pytorch_lightning/plugins/precision/deepspeed_precision.py b/pytorch_lightning/plugins/precision/deepspeed_precision.py index bd92607fd3b17..20ad7f9895891 100644 --- a/pytorch_lightning/plugins/precision/deepspeed_precision.py +++ b/pytorch_lightning/plugins/precision/deepspeed_precision.py @@ -21,9 +21,13 @@ from pytorch_lightning.plugins.precision.precision_plugin import PrecisionPlugin from pytorch_lightning.utilities import GradClipAlgorithmType from pytorch_lightning.utilities.exceptions import MisconfigurationException +from pytorch_lightning.utilities.imports import _DEEPSPEED_AVAILABLE from pytorch_lightning.utilities.model_helpers import is_overridden from pytorch_lightning.utilities.warnings import WarningCache +if _DEEPSPEED_AVAILABLE: + from deepspeed import DeepSpeedEngine + warning_cache = WarningCache() @@ -40,7 +44,7 @@ def backward(self, model: "pl.LightningModule", closure_loss: Tensor, *args: Any "You have overridden the `LightningModule.backward` hook but it will be ignored since DeepSpeed handles" " the backward logic internally." ) - deepspeed_engine = model.trainer.model + deepspeed_engine: DeepSpeedEngine = model.trainer.model deepspeed_engine.backward(closure_loss, *args, **kwargs) def _run_backward(self, tensor: Tensor, model: Module, *args: Any, **kwargs: Any) -> None: diff --git a/tests/lite/test_lite_api.py b/tests/lite/test_lite_api.py new file mode 100644 index 0000000000000..66ea03a162f46 --- /dev/null +++ b/tests/lite/test_lite_api.py @@ -0,0 +1,191 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from copy import deepcopy +from unittest import mock + +import pytest +import torch +import torch.distributed +import torch.nn.functional +from torch import nn +from torch.utils.data import DataLoader + +from pytorch_lightning import seed_everything +from pytorch_lightning.lite import LightningLite +from pytorch_lightning.plugins import DeepSpeedPlugin +from pytorch_lightning.utilities.exceptions import MisconfigurationException +from tests.helpers.boring_model import RandomDataset +from tests.helpers.runif import RunIf + + +class BoringModel(nn.Module): + def __init__(self): + super().__init__() + self.layer = torch.nn.Linear(32, 2, bias=False) + + def forward(self, x): + x = self.layer(x) + return torch.nn.functional.mse_loss(x, torch.ones_like(x)) + + +def configure_optimizers(module: nn.Module): + return torch.optim.SGD(module.parameters(), lr=0.0001) + + +def configure_optimizers_schedulers(module: nn.Module): + optimizer = torch.optim.SGD(module.parameters(), lr=0.1) + lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1) + return [optimizer], [lr_scheduler] + + +def test_lightning_lite_setup(): + class LiteRunner(LightningLite): + def run(self, pass_model: bool = True): + model = BoringModel() + optimizer = configure_optimizers(model) + model_lite, optimizer_lite = self.setup(model, optimizer) + if pass_model: + self.setup(model_lite, optimizer) + else: + self.setup(model, optimizer_lite) + + with pytest.raises(MisconfigurationException, match="A module should be passed only once to the"): + runner = LiteRunner() + runner.run() + + with pytest.raises(MisconfigurationException, match="An optimizer should be passed only once to the"): + runner = LiteRunner() + runner.run(pass_model=False) + + +def test_lightning_lite_setup_dataloaders(): + class LiteRunner(LightningLite): + def run(self): + + dataloader = DataLoader(RandomDataset(32, 64)) + dataloader_lite = self.setup_dataloaders(dataloader) + dataloader_lite = self.setup_dataloaders(dataloader_lite) + + with pytest.raises(MisconfigurationException, match="A dataloader should be passed only once to the"): + runner = LiteRunner() + runner.run() + + +def test_lightning_lite_track_model_setup(): + class LiteRunner(LightningLite): + def run(self): + model = BoringModel() + optimizer = configure_optimizers(model) + self.setup(model, optimizer) + assert not self._is_using_multiple_models + + model = BoringModel() + optimizer = configure_optimizers(model) + self.setup(model, optimizer) + assert self._is_using_multiple_models + + runner = LiteRunner() + runner.run() + + +def test_lightning_lite_deepspeed_backward(): + with mock.patch("pytorch_lightning.plugins.DeepSpeedPlugin.setup_distributed", lambda x: x): + + class LiteRunner(LightningLite): + def run(self): + def fn(*args): + return args + + self._strategy._setup_model_and_optimizer = fn + model = BoringModel() + optimizer = configure_optimizers(model) + self.setup(model, optimizer) + + model = BoringModel() + optimizer = configure_optimizers(model) + self.setup(model, optimizer) + + x = model(torch.randn(1, 32)) + loss = x.sum() + self.backward(loss) + + with pytest.raises(MisconfigurationException, match="please provide the model used to perform"): + runner = LiteRunner(strategy="deepspeed") + runner.run() + + +@RunIf(min_gpus=2, deepspeed=True, special=True) +def test_deepspeed_multiple_models(): + class LiteRunner(LightningLite): + def run(self): + model = BoringModel() + optimizer = configure_optimizers(model) + model, optimizer = self.setup(model, optimizer) + state_dict = deepcopy(model.state_dict()) + + for _ in range(2): + optimizer.zero_grad() + x = model(torch.randn(1, 32).to(self.device)) + loss = x.sum() + self.backward(loss, model=model) + optimizer.step() + + for mw_b, mw_a in zip(state_dict.values(), model.state_dict().values()): + assert not torch.equal(mw_b, mw_a) + + seed_everything(42) + model_1 = BoringModel() + optimizer_1 = configure_optimizers(model_1) + + seed_everything(42) + model_2 = BoringModel() + optimizer_2 = configure_optimizers(model_2) + + for mw_1, mw_2 in zip(model_1.state_dict().values(), model_2.state_dict().values()): + assert torch.equal(mw_1, mw_2) + + model_1, optimizer_1 = self.setup(model_1, optimizer_1) + model_2, optimizer_2 = self.setup(model_2, optimizer_2) + + seed_everything(42) + data_list = [] + for _ in range(2): + optimizer_1.zero_grad() + data = torch.randn(1, 32).to(self.device) + data_list.append(data) + x = model_1(data) + loss = x.sum() + self.backward(loss, model=model_1) + optimizer_1.step() + + for mw_1, mw_2 in zip(model_1.state_dict().values(), model_2.state_dict().values()): + assert not torch.equal(mw_1, mw_2) + + for data in data_list: + optimizer_2.zero_grad() + x = model_2(data) + loss = x.sum() + self.backward(loss, model=model_2) + optimizer_2.step() + + for mw_1, mw_2 in zip(model_1.state_dict().values(), model_2.state_dict().values()): + assert torch.equal(mw_1, mw_2) + + # Verify collectives works as expected + ranks = self.all_gather(torch.tensor([self.local_rank]).to(self.device)) + assert torch.equal(ranks.cpu(), torch.tensor([[0], [1]])) + assert self.broadcast(True) + assert self.is_global_zero == (self.local_rank == 0) + + LiteRunner(strategy=DeepSpeedPlugin(stage=3, logging_batch_size_per_gpu=1), devices=2, accelerator="gpu").run() From c74b2310c4dcad0fb2260ef02f4252b7796fa895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 16:53:42 +0200 Subject: [PATCH 171/331] merge all tests together --- tests/lite/test_lite.py | 165 ++++++++++++++++++++++++++++++- tests/lite/test_lite_api.py | 191 ------------------------------------ 2 files changed, 164 insertions(+), 192 deletions(-) delete mode 100644 tests/lite/test_lite_api.py diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 8315b08a7066f..9eebff495ec0c 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -11,18 +11,25 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from copy import deepcopy from unittest import mock from unittest.mock import Mock, PropertyMock import pytest import torch +import torch.distributed +import torch.nn.functional +from torch import nn from torch.utils.data import DataLoader, DistributedSampler, Sampler +from pytorch_lightning import seed_everything from pytorch_lightning.accelerators import Accelerator from pytorch_lightning.lite import LightningLite from pytorch_lightning.lite.wrappers import _LiteDataLoader -from pytorch_lightning.plugins import PrecisionPlugin, TrainingTypePlugin +from pytorch_lightning.plugins import DeepSpeedPlugin, PrecisionPlugin, TrainingTypePlugin from pytorch_lightning.utilities.exceptions import MisconfigurationException +from tests.helpers.boring_model import RandomDataset from tests.helpers.runif import RunIf @@ -31,6 +38,20 @@ def run(self): pass +class BoringModel(nn.Module): + def __init__(self): + super().__init__() + self.layer = torch.nn.Linear(32, 2, bias=False) + + def forward(self, x): + x = self.layer(x) + return torch.nn.functional.mse_loss(x, torch.ones_like(x)) + + +def configure_optimizers(module: nn.Module): + return torch.optim.SGD(module.parameters(), lr=0.0001) + + @pytest.mark.parametrize("accelerator", ["coconut", Mock(spec=Accelerator)]) def test_unsupported_accelerator(accelerator): with pytest.raises(MisconfigurationException, match=f"`accelerator={repr(accelerator)}` is not a valid choice"): @@ -169,3 +190,145 @@ def test_backward(): loss = Mock() lite.backward(loss, "arg", keyword="kwarg") lite._precision_plugin._run_backward.assert_called_with(loss, None, "arg", keyword="kwarg") + + +def test_lightning_lite_setup(): + class LiteRunner(LightningLite): + def run(self, pass_model: bool = True): + model = BoringModel() + optimizer = configure_optimizers(model) + model_lite, optimizer_lite = self.setup(model, optimizer) + if pass_model: + self.setup(model_lite, optimizer) + else: + self.setup(model, optimizer_lite) + + with pytest.raises(MisconfigurationException, match="A module should be passed only once to the"): + runner = LiteRunner() + runner.run() + + with pytest.raises(MisconfigurationException, match="An optimizer should be passed only once to the"): + runner = LiteRunner() + runner.run(pass_model=False) + + +def test_lightning_lite_setup_dataloaders(): + class LiteRunner(LightningLite): + def run(self): + + dataloader = DataLoader(RandomDataset(32, 64)) + dataloader_lite = self.setup_dataloaders(dataloader) + dataloader_lite = self.setup_dataloaders(dataloader_lite) + + with pytest.raises(MisconfigurationException, match="A dataloader should be passed only once to the"): + runner = LiteRunner() + runner.run() + + +def test_lightning_lite_track_model_setup(): + class LiteRunner(LightningLite): + def run(self): + model = BoringModel() + optimizer = configure_optimizers(model) + self.setup(model, optimizer) + assert not self._is_using_multiple_models + + model = BoringModel() + optimizer = configure_optimizers(model) + self.setup(model, optimizer) + assert self._is_using_multiple_models + + runner = LiteRunner() + runner.run() + + +def test_lightning_lite_deepspeed_backward(): + with mock.patch("pytorch_lightning.plugins.DeepSpeedPlugin.setup_distributed", lambda x: x): + + class LiteRunner(LightningLite): + def run(self): + def fn(*args): + return args + + self._strategy._setup_model_and_optimizer = fn + model = BoringModel() + optimizer = configure_optimizers(model) + self.setup(model, optimizer) + + model = BoringModel() + optimizer = configure_optimizers(model) + self.setup(model, optimizer) + + x = model(torch.randn(1, 32)) + loss = x.sum() + self.backward(loss) + + with pytest.raises(MisconfigurationException, match="please provide the model used to perform"): + runner = LiteRunner(strategy="deepspeed") + runner.run() + + +@RunIf(min_gpus=2, deepspeed=True, special=True) +def test_deepspeed_multiple_models(): + class LiteRunner(LightningLite): + def run(self): + model = BoringModel() + optimizer = configure_optimizers(model) + model, optimizer = self.setup(model, optimizer) + state_dict = deepcopy(model.state_dict()) + + for _ in range(2): + optimizer.zero_grad() + x = model(torch.randn(1, 32).to(self.device)) + loss = x.sum() + self.backward(loss, model=model) + optimizer.step() + + for mw_b, mw_a in zip(state_dict.values(), model.state_dict().values()): + assert not torch.equal(mw_b, mw_a) + + seed_everything(42) + model_1 = BoringModel() + optimizer_1 = configure_optimizers(model_1) + + seed_everything(42) + model_2 = BoringModel() + optimizer_2 = configure_optimizers(model_2) + + for mw_1, mw_2 in zip(model_1.state_dict().values(), model_2.state_dict().values()): + assert torch.equal(mw_1, mw_2) + + model_1, optimizer_1 = self.setup(model_1, optimizer_1) + model_2, optimizer_2 = self.setup(model_2, optimizer_2) + + seed_everything(42) + data_list = [] + for _ in range(2): + optimizer_1.zero_grad() + data = torch.randn(1, 32).to(self.device) + data_list.append(data) + x = model_1(data) + loss = x.sum() + self.backward(loss, model=model_1) + optimizer_1.step() + + for mw_1, mw_2 in zip(model_1.state_dict().values(), model_2.state_dict().values()): + assert not torch.equal(mw_1, mw_2) + + for data in data_list: + optimizer_2.zero_grad() + x = model_2(data) + loss = x.sum() + self.backward(loss, model=model_2) + optimizer_2.step() + + for mw_1, mw_2 in zip(model_1.state_dict().values(), model_2.state_dict().values()): + assert torch.equal(mw_1, mw_2) + + # Verify collectives works as expected + ranks = self.all_gather(torch.tensor([self.local_rank]).to(self.device)) + assert torch.equal(ranks.cpu(), torch.tensor([[0], [1]])) + assert self.broadcast(True) + assert self.is_global_zero == (self.local_rank == 0) + + LiteRunner(strategy=DeepSpeedPlugin(stage=3, logging_batch_size_per_gpu=1), devices=2, accelerator="gpu").run() diff --git a/tests/lite/test_lite_api.py b/tests/lite/test_lite_api.py deleted file mode 100644 index 66ea03a162f46..0000000000000 --- a/tests/lite/test_lite_api.py +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright The PyTorch Lightning team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from copy import deepcopy -from unittest import mock - -import pytest -import torch -import torch.distributed -import torch.nn.functional -from torch import nn -from torch.utils.data import DataLoader - -from pytorch_lightning import seed_everything -from pytorch_lightning.lite import LightningLite -from pytorch_lightning.plugins import DeepSpeedPlugin -from pytorch_lightning.utilities.exceptions import MisconfigurationException -from tests.helpers.boring_model import RandomDataset -from tests.helpers.runif import RunIf - - -class BoringModel(nn.Module): - def __init__(self): - super().__init__() - self.layer = torch.nn.Linear(32, 2, bias=False) - - def forward(self, x): - x = self.layer(x) - return torch.nn.functional.mse_loss(x, torch.ones_like(x)) - - -def configure_optimizers(module: nn.Module): - return torch.optim.SGD(module.parameters(), lr=0.0001) - - -def configure_optimizers_schedulers(module: nn.Module): - optimizer = torch.optim.SGD(module.parameters(), lr=0.1) - lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1) - return [optimizer], [lr_scheduler] - - -def test_lightning_lite_setup(): - class LiteRunner(LightningLite): - def run(self, pass_model: bool = True): - model = BoringModel() - optimizer = configure_optimizers(model) - model_lite, optimizer_lite = self.setup(model, optimizer) - if pass_model: - self.setup(model_lite, optimizer) - else: - self.setup(model, optimizer_lite) - - with pytest.raises(MisconfigurationException, match="A module should be passed only once to the"): - runner = LiteRunner() - runner.run() - - with pytest.raises(MisconfigurationException, match="An optimizer should be passed only once to the"): - runner = LiteRunner() - runner.run(pass_model=False) - - -def test_lightning_lite_setup_dataloaders(): - class LiteRunner(LightningLite): - def run(self): - - dataloader = DataLoader(RandomDataset(32, 64)) - dataloader_lite = self.setup_dataloaders(dataloader) - dataloader_lite = self.setup_dataloaders(dataloader_lite) - - with pytest.raises(MisconfigurationException, match="A dataloader should be passed only once to the"): - runner = LiteRunner() - runner.run() - - -def test_lightning_lite_track_model_setup(): - class LiteRunner(LightningLite): - def run(self): - model = BoringModel() - optimizer = configure_optimizers(model) - self.setup(model, optimizer) - assert not self._is_using_multiple_models - - model = BoringModel() - optimizer = configure_optimizers(model) - self.setup(model, optimizer) - assert self._is_using_multiple_models - - runner = LiteRunner() - runner.run() - - -def test_lightning_lite_deepspeed_backward(): - with mock.patch("pytorch_lightning.plugins.DeepSpeedPlugin.setup_distributed", lambda x: x): - - class LiteRunner(LightningLite): - def run(self): - def fn(*args): - return args - - self._strategy._setup_model_and_optimizer = fn - model = BoringModel() - optimizer = configure_optimizers(model) - self.setup(model, optimizer) - - model = BoringModel() - optimizer = configure_optimizers(model) - self.setup(model, optimizer) - - x = model(torch.randn(1, 32)) - loss = x.sum() - self.backward(loss) - - with pytest.raises(MisconfigurationException, match="please provide the model used to perform"): - runner = LiteRunner(strategy="deepspeed") - runner.run() - - -@RunIf(min_gpus=2, deepspeed=True, special=True) -def test_deepspeed_multiple_models(): - class LiteRunner(LightningLite): - def run(self): - model = BoringModel() - optimizer = configure_optimizers(model) - model, optimizer = self.setup(model, optimizer) - state_dict = deepcopy(model.state_dict()) - - for _ in range(2): - optimizer.zero_grad() - x = model(torch.randn(1, 32).to(self.device)) - loss = x.sum() - self.backward(loss, model=model) - optimizer.step() - - for mw_b, mw_a in zip(state_dict.values(), model.state_dict().values()): - assert not torch.equal(mw_b, mw_a) - - seed_everything(42) - model_1 = BoringModel() - optimizer_1 = configure_optimizers(model_1) - - seed_everything(42) - model_2 = BoringModel() - optimizer_2 = configure_optimizers(model_2) - - for mw_1, mw_2 in zip(model_1.state_dict().values(), model_2.state_dict().values()): - assert torch.equal(mw_1, mw_2) - - model_1, optimizer_1 = self.setup(model_1, optimizer_1) - model_2, optimizer_2 = self.setup(model_2, optimizer_2) - - seed_everything(42) - data_list = [] - for _ in range(2): - optimizer_1.zero_grad() - data = torch.randn(1, 32).to(self.device) - data_list.append(data) - x = model_1(data) - loss = x.sum() - self.backward(loss, model=model_1) - optimizer_1.step() - - for mw_1, mw_2 in zip(model_1.state_dict().values(), model_2.state_dict().values()): - assert not torch.equal(mw_1, mw_2) - - for data in data_list: - optimizer_2.zero_grad() - x = model_2(data) - loss = x.sum() - self.backward(loss, model=model_2) - optimizer_2.step() - - for mw_1, mw_2 in zip(model_1.state_dict().values(), model_2.state_dict().values()): - assert torch.equal(mw_1, mw_2) - - # Verify collectives works as expected - ranks = self.all_gather(torch.tensor([self.local_rank]).to(self.device)) - assert torch.equal(ranks.cpu(), torch.tensor([[0], [1]])) - assert self.broadcast(True) - assert self.is_global_zero == (self.local_rank == 0) - - LiteRunner(strategy=DeepSpeedPlugin(stage=3, logging_batch_size_per_gpu=1), devices=2, accelerator="gpu").run() From b4310ad10ae9c94da50e52488fb966b69377fb37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 17:05:25 +0200 Subject: [PATCH 172/331] _num_models checks --- pytorch_lightning/lite/lite.py | 10 +++------- tests/lite/test_lite.py | 8 +++++--- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 3c094f3924961..8c4ba1577e89a 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -107,7 +107,7 @@ def __init__( # wrap the run method so we can inject setup logic or spawn processes for the user setattr(self, "run", self._run_wrapper(self.run)) - self._number_of_models: int = 0 + self._num_models: int = 0 @property def device(self) -> torch.device: @@ -142,10 +142,6 @@ def is_global_zero(self) -> bool: """Wether this rank is rank zero.""" return self._strategy.is_global_zero - @property - def _is_using_multiple_models(self) -> bool: - return self._number_of_models > 1 - @abstractmethod def run(self, *args: Any, **kwargs: Any) -> Any: """All the code inside this run method gets accelerated by Lite. @@ -192,7 +188,7 @@ def setup( model, optimizers = self._setup_model_and_optimizers(model, optimizers) optimizers = optimizers[0] if len(optimizers) == 1 else optimizers - self._number_of_models += 1 + self._num_models += 1 return model, optimizers def setup_dataloaders( @@ -264,7 +260,7 @@ def backward(self, tensor: Tensor, *args: Any, model: Optional[_LiteModule] = No *args: Optional positional arguments passed to the underlying backward function. **kwargs: Optional named keyword arguments passed to the underlying backward function. """ - if self._is_using_multiple_models and isinstance(self._strategy, DeepSpeedPlugin): + if self._num_models > 0 and isinstance(self._strategy, DeepSpeedPlugin): if not isinstance(model, _LiteModule): raise MisconfigurationException( "When using multiple models + deepspeed, please provide the model used to perform the optimization." diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 9eebff495ec0c..9f734a0b3a9fd 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -218,7 +218,7 @@ def run(self): dataloader = DataLoader(RandomDataset(32, 64)) dataloader_lite = self.setup_dataloaders(dataloader) - dataloader_lite = self.setup_dataloaders(dataloader_lite) + _ = self.setup_dataloaders(dataloader_lite) with pytest.raises(MisconfigurationException, match="A dataloader should be passed only once to the"): runner = LiteRunner() @@ -230,13 +230,15 @@ class LiteRunner(LightningLite): def run(self): model = BoringModel() optimizer = configure_optimizers(model) + + assert self._num_models == 0 self.setup(model, optimizer) - assert not self._is_using_multiple_models + assert self._num_models == 1 model = BoringModel() optimizer = configure_optimizers(model) self.setup(model, optimizer) - assert self._is_using_multiple_models + assert self._num_models == 2 runner = LiteRunner() runner.run() From 143aef882a17e2d3eaeb86e696736328ca4d7bc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 17:05:35 +0200 Subject: [PATCH 173/331] use decorator for patching --- tests/lite/test_lite.py | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 9f734a0b3a9fd..ebb99b423bcdf 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -244,30 +244,29 @@ def run(self): runner.run() +@mock.patch("pytorch_lightning.plugins.DeepSpeedPlugin.setup_distributed", lambda x: x) def test_lightning_lite_deepspeed_backward(): - with mock.patch("pytorch_lightning.plugins.DeepSpeedPlugin.setup_distributed", lambda x: x): - - class LiteRunner(LightningLite): - def run(self): - def fn(*args): - return args + class LiteRunner(LightningLite): + def run(self): + def fn(*args): + return args - self._strategy._setup_model_and_optimizer = fn - model = BoringModel() - optimizer = configure_optimizers(model) - self.setup(model, optimizer) + self._strategy._setup_model_and_optimizer = fn + model = BoringModel() + optimizer = configure_optimizers(model) + self.setup(model, optimizer) - model = BoringModel() - optimizer = configure_optimizers(model) - self.setup(model, optimizer) + model = BoringModel() + optimizer = configure_optimizers(model) + self.setup(model, optimizer) - x = model(torch.randn(1, 32)) - loss = x.sum() - self.backward(loss) + x = model(torch.randn(1, 32)) + loss = x.sum() + self.backward(loss) - with pytest.raises(MisconfigurationException, match="please provide the model used to perform"): - runner = LiteRunner(strategy="deepspeed") - runner.run() + with pytest.raises(MisconfigurationException, match="please provide the model used to perform"): + runner = LiteRunner(strategy="deepspeed") + runner.run() @RunIf(min_gpus=2, deepspeed=True, special=True) From 08cd1221eb565cb7f7680663588ec9d198287046 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 17:09:49 +0200 Subject: [PATCH 174/331] docs for deepspeed special case --- pytorch_lightning/lite/lite.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 8c4ba1577e89a..614de6393ed34 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -258,7 +258,12 @@ def backward(self, tensor: Tensor, *args: Any, model: Optional[_LiteModule] = No Args: tensor: The tensor (loss) to back-propagate gradients from. *args: Optional positional arguments passed to the underlying backward function. + model: Optional model instance for plugins that require the model for backward(). **kwargs: Optional named keyword arguments passed to the underlying backward function. + + Note: + When using ``strategy='deepspeed'`` and multiple models were setup, it is required to pass in the + model as argument here. """ if self._num_models > 0 and isinstance(self._strategy, DeepSpeedPlugin): if not isinstance(model, _LiteModule): @@ -266,8 +271,8 @@ def backward(self, tensor: Tensor, *args: Any, model: Optional[_LiteModule] = No "When using multiple models + deepspeed, please provide the model used to perform the optimization." ) - # requires to attach the current deepSpeed engine for the `optimizer.step` call. - self._strategy.model = model._module + # requires to attach the current `DeepSpeedEngine` for the `_LiteOptimizer.step` call. + self._strategy.model = model.module self._precision_plugin._run_backward(tensor, self._strategy.model, *args, **kwargs) From f8a0f45a84dfc5c32be132c7855bf7efa3aaee4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 17:32:41 +0200 Subject: [PATCH 175/331] rename wrapper for sharded context --- pytorch_lightning/lite/lite.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 614de6393ed34..9a2936aeff65b 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -399,15 +399,15 @@ def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> Any: self._set_plugin_specific_precision_variables() self._accelerator.setup_environment() - run_fn = partial(self._run_method_wrapper, run_method, *args, **kwargs) + # apply sharded context to prevent OOM + run_method = partial(self._run_with_sharded_context, run_method) if isinstance(self._strategy, DDPSpawnPlugin): - return self._strategy.spawn(run_fn) + return self._strategy.spawn(run_method, *args, **kwargs) else: - return run_fn() + return run_method(*args, **kwargs) - def _run_method_wrapper(self, run_method: Callable, *args: Any, **kwargs: Any) -> Any: - # requires to apply sharded context to prevent OOM + def _run_with_sharded_context(self, run_method: Callable, *args: Any, **kwargs: Any) -> Any: with self._strategy.model_sharded_context(): return run_method(*args, **kwargs) From 0b30c8e63f80fd09857aa3b25adfb6237813e654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 17:37:47 +0200 Subject: [PATCH 176/331] add todo --- tests/lite/test_lite.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index ebb99b423bcdf..76f66662a2c83 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -244,6 +244,7 @@ def run(self): runner.run() +# TODO: This test does not assert any functionality: use Mock to assert how DeepSpeedPlugin gets called @mock.patch("pytorch_lightning.plugins.DeepSpeedPlugin.setup_distributed", lambda x: x) def test_lightning_lite_deepspeed_backward(): class LiteRunner(LightningLite): From 538f6de9665bdc60170f56a793aa9269322f7b22 Mon Sep 17 00:00:00 2001 From: tchaton Date: Thu, 21 Oct 2021 17:27:15 +0100 Subject: [PATCH 177/331] improve typing --- pytorch_lightning/lite/lite.py | 52 ++++++++++++++++-------------- pytorch_lightning/lite/wrappers.py | 14 ++++---- tests/lite/test_lite.py | 7 ++++ 3 files changed, 42 insertions(+), 31 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 9a2936aeff65b..8ca76ffbc7ce7 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -17,7 +17,7 @@ from contextlib import contextmanager from functools import partial from pathlib import Path -from typing import Any, Dict, Generator, List, Optional, Sequence, Tuple, Union +from typing import Any, Dict, Generator, Iterable, List, Optional, Sequence, Tuple, Union import torch import torch.nn as nn @@ -187,13 +187,13 @@ def setup( param_group["params"] = [mapping.get(p, p) for p in param_group["params"]] model, optimizers = self._setup_model_and_optimizers(model, optimizers) - optimizers = optimizers[0] if len(optimizers) == 1 else optimizers + optimizers = optimizers[0] if isinstance(optimizers, Sequence) and len(optimizers) == 1 else optimizers self._num_models += 1 return model, optimizers def setup_dataloaders( self, *dataloaders: DataLoader, replace_sampler: bool = True, move_to_device: bool = True - ) -> Union[DataLoader, List[DataLoader]]: + ) -> Union[DataLoader, List[DataLoader], Iterable]: """Setup one or multiple dataloaders for accelerated training. If you need different settings for each dataloader, call this method individually for each one. @@ -218,8 +218,8 @@ def setup_dataloaders( return dataloaders def _setup_dataloader( - self, dataloader: DataLoader, replace_sampler: bool = True, move_to_device: bool = True - ) -> DataLoader: + self, dataloader: Union[Iterable, DataLoader], replace_sampler: bool = True, move_to_device: bool = True + ) -> Union[Iterable, DataLoader]: """Setup a single dataloader for accelerated training. Args: @@ -233,23 +233,23 @@ def _setup_dataloader( Returns: The wrapped dataloader. """ - sampler = dataloader.sampler - if replace_sampler and self._requires_distributed_sampler(dataloader): - if not isinstance(dataloader.sampler, (SequentialSampler, RandomSampler)): - raise MisconfigurationException( - "You seem to have configured a sampler in your DataLoader. This will be replaced " - " by `DistributedSampler` since `replace_sampler_ddp` is True and you are using" - " distributed training. Either remove the sampler from your DataLoader or set" - " `replace_sampler=False` if you want to use your custom sampler." - ) - sampler = self._get_distributed_sampler(dataloader, **self._strategy.distributed_sampler_kwargs) - - kwargs = TrainerDataLoadingMixin._get_dataloader_init_kwargs(dataloader, sampler) - device = self.device if move_to_device else None - if isinstance(self._strategy, TPUSpawnPlugin): - dataloader = DataLoader(**kwargs) - else: - dataloader = _LiteDataLoader(device=device, **kwargs) + if isinstance(dataloader, DataLoader): + if replace_sampler and self._requires_distributed_sampler(dataloader): + if not isinstance(dataloader.sampler, (SequentialSampler, RandomSampler)): + raise MisconfigurationException( + "You seem to have configured a sampler in your DataLoader. This will be replaced " + " by `DistributedSampler` since `replace_sampler_ddp` is True and you are using" + " distributed training. Either remove the sampler from your DataLoader or set" + " `replace_sampler=False` if you want to use your custom sampler." + ) + sampler = self._get_distributed_sampler(dataloader, **self._strategy.distributed_sampler_kwargs) + + kwargs = TrainerDataLoadingMixin._get_dataloader_init_kwargs(dataloader, sampler) + device = self.device if move_to_device else None + if isinstance(self._strategy, TPUSpawnPlugin): + dataloader = DataLoader(**kwargs) + else: + dataloader = _LiteDataLoader(device=device, **kwargs) return self._strategy.process_dataloader(dataloader) def backward(self, tensor: Tensor, *args: Any, model: Optional[_LiteModule] = None, **kwargs: Any) -> None: @@ -274,6 +274,7 @@ def backward(self, tensor: Tensor, *args: Any, model: Optional[_LiteModule] = No # requires to attach the current `DeepSpeedEngine` for the `_LiteOptimizer.step` call. self._strategy.model = model.module + assert self._strategy.model self._precision_plugin._run_backward(tensor, self._strategy.model, *args, **kwargs) @contextmanager @@ -347,7 +348,7 @@ def reduce_decision(self, decision: bool) -> bool: def all_gather( self, data: Union[torch.Tensor, Dict, List, Tuple], group: Optional[Any] = None, sync_grads: bool = False - ): + ) -> Union[torch.Tensor, Dict, List, Tuple]: r""" Gather tensors or collections of tensors from multiple processes. @@ -428,7 +429,7 @@ def _setup_model_and_optimizers( self, model: nn.Module, optimizers: List[Optimizer], - ) -> Tuple[_LiteModule, List[_LiteOptimizer]]: + ) -> Tuple[_LiteModule, Union[_LiteOptimizer, List[_LiteOptimizer]]]: # Let accelerator/plugin wrap and connect the models and optimizers [model], optimizers = self._strategy._setup_models_and_optimizers([model], optimizers) model = _LiteModule(model, self._accelerator) @@ -502,3 +503,6 @@ def _validate_setup_dataloaders(*dataloaders: Union[DataLoader, List[DataLoader] raise MisconfigurationException( "A dataloader should be passed only once to the ``setup_dataloaders`` method" ) + + if any(not isinstance(dl, DataLoader) for dl in dataloaders): + raise MisconfigurationException("Only PyTorch DataLoader are currently supported.") diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 9ab8703d2b529..34455f4ca091c 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Callable, Generator, Iterator, Optional, Union +from typing import Any, Callable, Dict, Generator, Iterator, List, Optional, Union import torch from torch import nn as nn @@ -48,27 +48,27 @@ def optimizer(self) -> Optimizer: return self._optimizer @property - def state(self): + def state(self) -> Dict[str, torch.Tensor]: return self._optimizer.state @state.setter - def state(self, state): + def state(self, state: Dict[str, torch.Tensor]) -> None: self._optimizer.state = state @property - def defaults(self): + def defaults(self) -> Dict[str, Any]: return self._optimizer.defaults @defaults.setter - def defaults(self, defaults): + def defaults(self, defaults: Dict[str, Any]) -> None: self._optimizer.defaults = defaults @property - def param_groups(self): + def param_groups(self) -> List[Dict[str, torch.Tensor]]: return self._optimizer.param_groups @param_groups.setter - def param_groups(self, param_groups): + def param_groups(self, param_groups: List[Dict[str, torch.Tensor]]) -> None: self._optimizer.param_groups = param_groups def step(self, closure: Optional[Callable] = None) -> None: diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 76f66662a2c83..c9b0f39acc9f2 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -84,6 +84,13 @@ def test_setup_dataloaders_return_type(): assert lite_dataloader1.dataset is dataset1 +def test_lite_with_iterable(): + """Test that the setup_dataloaders method fails when provided with an iterable.""" + lite = EmptyLite() + with pytest.raises(MisconfigurationException, match="Only PyTorch DataLoader are currently supported"): + lite.setup_dataloaders(range(2)) + + @mock.patch( "pytorch_lightning.lite.lite.LightningLite.device", new_callable=PropertyMock, From ae7af7877cc8a9ab4854ef5f0451a348b7c61c9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 22:59:21 +0200 Subject: [PATCH 178/331] delete debug examples --- pl_examples/debug/__init__.py | 0 pl_examples/debug/gan/__init__.py | 0 pl_examples/debug/gan/gan_example.py | 178 ---------------------- pl_examples/debug/gan/models.py | 78 ---------- pl_examples/debug/gan/run_examples.py | 15 -- pl_examples/debug/simple/__init__.py | 0 pl_examples/debug/simple/mnist_example.py | 178 ---------------------- 7 files changed, 449 deletions(-) delete mode 100644 pl_examples/debug/__init__.py delete mode 100644 pl_examples/debug/gan/__init__.py delete mode 100644 pl_examples/debug/gan/gan_example.py delete mode 100644 pl_examples/debug/gan/models.py delete mode 100644 pl_examples/debug/gan/run_examples.py delete mode 100644 pl_examples/debug/simple/__init__.py delete mode 100644 pl_examples/debug/simple/mnist_example.py diff --git a/pl_examples/debug/__init__.py b/pl_examples/debug/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/pl_examples/debug/gan/__init__.py b/pl_examples/debug/gan/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/pl_examples/debug/gan/gan_example.py b/pl_examples/debug/gan/gan_example.py deleted file mode 100644 index 2232ce339f584..0000000000000 --- a/pl_examples/debug/gan/gan_example.py +++ /dev/null @@ -1,178 +0,0 @@ -""" -DCGAN - Adapted from pytorch/examples - -Launch it with this command: - -python -m torch.distributed.run --nproc_per_node=2 gan_example.py - -""" - -import argparse -import os - -import torch -import torch.nn as nn -import torch.nn.parallel -import torch.optim as optim -import torch.utils.data -import torchvision.datasets as dset -import torchvision.transforms as transforms -import torchvision.utils as vutils - -from pl_examples.debug.gan.models import Discriminator, Generator, weights_init -from pytorch_lightning import seed_everything -from pytorch_lightning.lite import LightningLite -from pytorch_lightning.lite.wrappers import _LiteModule, _LiteOptimizer - -parser = argparse.ArgumentParser() -parser.add_argument("--workers", type=int, help="number of data loading workers", default=0) -parser.add_argument("--batchSize", type=int, default=64, help="input batch size") -parser.add_argument( - "--imageSize", - type=int, - default=64, - help="the height / width of the input image to network", -) -parser.add_argument("--niter", type=int, default=25, help="number of epochs to train for") -parser.add_argument("--lr", type=float, default=0.0002, help="learning rate, default=0.0002") -parser.add_argument("--beta1", type=float, default=0.5, help="beta1 for adam. default=0.5") -parser.add_argument("--ngpu", type=int, default=1, help="number of GPUs to use") -parser.add_argument("--netG", default="", help="path to netG (to continue training)") -parser.add_argument("--netD", default="", help="path to netD (to continue training)") -parser.add_argument("--outf", default="./lightning_logs", help="folder to output images and model checkpoints") -parser.add_argument("--local_rank", type=int, default=0) - -opt, _ = parser.parse_known_args() -os.makedirs(opt.outf, exist_ok=True) -ngpu = int(opt.ngpu) - -nz = 100 - - -class GANTrainer(LightningLite): - def run(self): - print("strategy: ", self._strategy) - print("precision plugin: ", self._precision_plugin) - seed_everything(123) - - # TODO: how do we handle this in Accelerator? - # torch.cuda.set_device(opt.local_rank) - # TODO: how do we handle this? - # os.environ["LOCAL_RANK"] = str(opt.local_rank) - # os.environ["NODE_RANK"] = str(opt.local_rank) - - if self.local_rank == 0: - dset.MNIST(root=".", download=True) - - self.barrier() - dataset = dset.MNIST( - root=".", - transform=transforms.Compose( - [ - transforms.Resize(opt.imageSize), - transforms.ToTensor(), - transforms.Normalize((0.5,), (0.5,)), - ] - ), - ) - dataloader = torch.utils.data.DataLoader( - dataset, batch_size=opt.batchSize, shuffle=True, num_workers=opt.workers - ) - - dataloader = self.setup_dataloaders(dataloader) - # assert isinstance(dataloader.sampler, DistributedSampler) - - netG = Generator() - netG.apply(weights_init) - - netD = Discriminator() - netD.apply(weights_init) - - # self.to_device(netG) - # self.to_device(netD) - - criterion = nn.BCELoss() - - fixed_noise = torch.randn(opt.batchSize, nz, 1, 1, device=self.device) - real_label = 1 - fake_label = 0 - - # setup optimizer - optimizerD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) - optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) - - netG, optimizerG = self.setup(netG, optimizerG) - netD, optimizerD = self.setup(netD, optimizerD) - - assert isinstance(optimizerG, _LiteOptimizer) - assert isinstance(netG, _LiteModule) - print("parameters dtype", next(netG.parameters()).dtype) - - for epoch in range(opt.niter): - for i, data in enumerate(dataloader, 0): - ############################ - # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z))) - ########################### - # train with real - netD.zero_grad() - real_cpu = data[0] - batch_size = real_cpu.size(0) - label = torch.full((batch_size,), real_label, dtype=real_cpu.dtype, device=self.device) - - output = netD(real_cpu) - errD_real = criterion(output, label) - self.backward(errD_real) - D_x = output.mean().item() - - # train with fake - noise = torch.randn(batch_size, nz, 1, 1, device=self.device) - fake = netG(noise) - label.fill_(fake_label) - output = netD(fake.detach()) - errD_fake = criterion(output, label) - self.backward(errD_fake) - D_G_z1 = output.mean().item() - errD = errD_real + errD_fake - optimizerD.step() - - ############################ - # (2) Update G network: maximize log(D(G(z))) - ########################### - netG.zero_grad() - label.fill_(real_label) # fake labels are real for generator cost - output = netD(fake) - errG = criterion(output, label) - self.backward(errG) - D_G_z2 = output.mean().item() - optimizerG.step() - - print( - "[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)): %.4f / %.4f" - % ( - epoch, - opt.niter, - i, - len(dataloader), - errD.item(), - errG.item(), - D_x, - D_G_z1, - D_G_z2, - ) - ) - if i % 100 == 0: - vutils.save_image(real_cpu, "%s/real_samples.png" % opt.outf, normalize=True) - fake = netG(fixed_noise) - vutils.save_image( - fake.detach(), - "%s/fake_samples_epoch_%03d.png" % (opt.outf, epoch), - normalize=True, - ) - # do checkpointing - torch.save(netG.state_dict(), "%s/netG_epoch_%d.pth" % (opt.outf, epoch)) - torch.save(netD.state_dict(), "%s/netD_epoch_%d.pth" % (opt.outf, epoch)) - - -if __name__ == "__main__": - gan = GANTrainer(accelerator="ddp", devices=2) - gan.run() diff --git a/pl_examples/debug/gan/models.py b/pl_examples/debug/gan/models.py deleted file mode 100644 index 5ccdec18aebc2..0000000000000 --- a/pl_examples/debug/gan/models.py +++ /dev/null @@ -1,78 +0,0 @@ -import torch -from torch import nn as nn - -nc = 1 -nz = 100 -ngf = 64 -ndf = 64 - - -def weights_init(m): - classname = m.__class__.__name__ - if classname.find("Conv") != -1: - torch.nn.init.normal_(m.weight, 0.0, 0.02) - elif classname.find("BatchNorm") != -1: - torch.nn.init.normal_(m.weight, 1.0, 0.02) - torch.nn.init.zeros_(m.bias) - - -class Generator(nn.Module): - def __init__(self): - super().__init__() - self.main = nn.Sequential( - # input is Z, going into a convolution - nn.ConvTranspose2d(nz, ngf * 8, 4, 1, 0, bias=False), - nn.BatchNorm2d(ngf * 8), - nn.ReLU(True), - # state size. (ngf*8) x 4 x 4 - nn.ConvTranspose2d(ngf * 8, ngf * 4, 4, 2, 1, bias=False), - nn.BatchNorm2d(ngf * 4), - nn.ReLU(True), - # state size. (ngf*4) x 8 x 8 - nn.ConvTranspose2d(ngf * 4, ngf * 2, 4, 2, 1, bias=False), - nn.BatchNorm2d(ngf * 2), - nn.ReLU(True), - # state size. (ngf*2) x 16 x 16 - nn.ConvTranspose2d(ngf * 2, ngf, 4, 2, 1, bias=False), - nn.BatchNorm2d(ngf), - nn.ReLU(True), - # state size. (ngf) x 32 x 32 - nn.ConvTranspose2d(ngf, nc, 4, 2, 1, bias=False), - nn.Tanh() - # state size. (nc) x 64 x 64 - ) - - def forward(self, input): - print("autocast enabled in generator: ", torch.is_autocast_enabled()) - return self.main(input) - - -class Discriminator(nn.Module): - def __init__(self): - super().__init__() - self.main = nn.Sequential( - # input is (nc) x 64 x 64 - nn.Conv2d(nc, ndf, 4, 2, 1, bias=False), - nn.LeakyReLU(0.2, inplace=True), - # state size. (ndf) x 32 x 32 - nn.Conv2d(ndf, ndf * 2, 4, 2, 1, bias=False), - nn.BatchNorm2d(ndf * 2), - nn.LeakyReLU(0.2, inplace=True), - # state size. (ndf*2) x 16 x 16 - nn.Conv2d(ndf * 2, ndf * 4, 4, 2, 1, bias=False), - nn.BatchNorm2d(ndf * 4), - nn.LeakyReLU(0.2, inplace=True), - # state size. (ndf*4) x 8 x 8 - nn.Conv2d(ndf * 4, ndf * 8, 4, 2, 1, bias=False), - nn.BatchNorm2d(ndf * 8), - nn.LeakyReLU(0.2, inplace=True), - # state size. (ndf*8) x 4 x 4 - nn.Conv2d(ndf * 8, 1, 4, 1, 0, bias=False), - nn.Sigmoid(), - ) - - def forward(self, input): - print("autocast enabled in discriminator: ", torch.is_autocast_enabled()) - output = self.main(input) - print("double precision: ", input.dtype == torch.double) - return output.view(-1, 1).squeeze(1) diff --git a/pl_examples/debug/gan/run_examples.py b/pl_examples/debug/gan/run_examples.py deleted file mode 100644 index 3cd724a0639e2..0000000000000 --- a/pl_examples/debug/gan/run_examples.py +++ /dev/null @@ -1,15 +0,0 @@ -import argparse - -from pl_examples.debug.gan.gan_example import GANTrainer - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--accelerator", type=str, default=None) - parser.add_argument("--strategy", type=str, default=None) - parser.add_argument("--gpus", type=int, default=None) - parser.add_argument("--devices", type=int, default=1) - parser.add_argument("--precision", type=int, default=32) - args = parser.parse_args() - - trainer = GANTrainer(**vars(args)) - trainer.run() diff --git a/pl_examples/debug/simple/__init__.py b/pl_examples/debug/simple/__init__.py deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/pl_examples/debug/simple/mnist_example.py b/pl_examples/debug/simple/mnist_example.py deleted file mode 100644 index 3158170bce057..0000000000000 --- a/pl_examples/debug/simple/mnist_example.py +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright The PyTorch Lightning team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import argparse - -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.optim as optim -from torch.optim.lr_scheduler import StepLR -from torch.utils.data import DistributedSampler -from torchvision import datasets, transforms - -from pytorch_lightning import seed_everything -from pytorch_lightning.lite import LightningLite - - -class Net(nn.Module): - def __init__(self): - super().__init__() - self.conv1 = nn.Conv2d(1, 32, 3, 1) - self.conv2 = nn.Conv2d(32, 64, 3, 1) - self.dropout1 = nn.Dropout(0.25) - self.dropout2 = nn.Dropout(0.5) - self.fc1 = nn.Linear(9216, 128) - self.fc2 = nn.Linear(128, 10) - - def forward(self, x): - x = self.conv1(x) - x = F.relu(x) - x = self.conv2(x) - x = F.relu(x) - x = F.max_pool2d(x, 2) - x = self.dropout1(x) - x = torch.flatten(x, 1) - x = self.fc1(x) - x = F.relu(x) - x = self.dropout2(x) - x = self.fc2(x) - output = F.log_softmax(x, dim=1) - return output - - -class MNIST(LightningLite): - - # when we enter run() here, distributed setup already took place - def run(self, args): - use_cuda = self.device.type == "cuda" - - seed_everything(args.seed) - - train_kwargs = {"batch_size": args.batch_size} - test_kwargs = {"batch_size": args.test_batch_size} - if use_cuda: - cuda_kwargs = {"num_workers": 1, "pin_memory": True, "shuffle": True} - train_kwargs.update(cuda_kwargs) - test_kwargs.update(cuda_kwargs) - - if self.local_rank == 0: - datasets.MNIST("../data", download=True) - - transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]) - dataset1 = datasets.MNIST("../data", train=True, transform=transform) - dataset2 = datasets.MNIST("../data", train=False, transform=transform) - train_loader = torch.utils.data.DataLoader(dataset1, **train_kwargs) - test_loader = torch.utils.data.DataLoader(dataset2, **test_kwargs) - - model = Net() - optimizer = optim.Adadelta(model.parameters(), lr=args.lr) - scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) - - train_loader, test_loader = self.setup_dataloaders(train_loader, test_loader) - assert isinstance(train_loader.sampler, DistributedSampler) - assert isinstance(test_loader.sampler, DistributedSampler) - model, optimizer = self.setup(model, optimizer) - - for epoch in range(1, args.epochs + 1): - self.train(args, model, train_loader, optimizer, epoch) - self.test(model, test_loader) - scheduler.step() - - if args.save_model: - torch.save(model.state_dict(), "mnist_cnn.pt") - - def train(self, args, model, train_loader, optimizer, epoch): - model.train() - for batch_idx, (data, target) in enumerate(train_loader): - # data, target = data.to(self.device), target.to(self.device) - optimizer.zero_grad() - output = model(data) - loss = F.nll_loss(output, target) - self.backward(loss) - optimizer.step() - if batch_idx % args.log_interval == 0: - self.print( - "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format( - epoch, - batch_idx * len(data), - len(train_loader.dataset), - 100.0 * batch_idx / len(train_loader), - loss.item(), - ) - ) - if args.dry_run: - break - - def test(self, model, test_loader): - model.eval() - test_loss = 0 - correct = 0 - with torch.no_grad(): - for i, (data, target) in enumerate(test_loader): - data, target = data.to(self.device), target.to(self.device) - output = model(data) - test_loss += F.nll_loss(output, target, reduction="sum").item() # sum up batch loss - pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability - correct += pred.eq(target.view_as(pred)).sum().item() - - test_loss /= len(test_loader.dataset) - - self.print( - "\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n".format( - test_loss, correct, len(test_loader.dataset), 100.0 * correct / len(test_loader.dataset) - ) - ) - - -def main(): - # Training settings - parser = argparse.ArgumentParser(description="PyTorch MNIST Example") - parser.add_argument( - "--batch-size", type=int, default=64, metavar="N", help="input batch size for training (default: 64)" - ) - parser.add_argument( - "--test-batch-size", type=int, default=1000, metavar="N", help="input batch size for testing (default: 1000)" - ) - parser.add_argument("--epochs", type=int, default=14, metavar="N", help="number of epochs to train (default: 14)") - parser.add_argument("--lr", type=float, default=1.0, metavar="LR", help="learning rate (default: 1.0)") - parser.add_argument("--gamma", type=float, default=0.7, metavar="M", help="Learning rate step gamma (default: 0.7)") - parser.add_argument("--dry-run", action="store_true", default=False, help="quickly check a single pass") - parser.add_argument("--seed", type=int, default=1, metavar="S", help="random seed (default: 1)") - parser.add_argument( - "--log-interval", - type=int, - default=10, - metavar="N", - help="how many batches to wait before logging training status", - ) - parser.add_argument("--save-model", action="store_true", default=False, help="For Saving the current Model") - parser.add_argument("--accelerator", type=str, default=None) - parser.add_argument("--strategy", type=str, default=None) - parser.add_argument("--gpus", type=int, default=None) - parser.add_argument("--devices", type=int, default=1) - parser.add_argument("--precision", type=int, default=32) - args = parser.parse_args() - - mnist = MNIST( - gpus=args.gpus, - devices=args.devices, - accelerator=args.accelerator, - strategy=args.strategy, - precision=args.precision, - ) - mnist.run(args) - - -if __name__ == "__main__": - main() From fadca442f4cc9aaf727cee335229be92853b04af Mon Sep 17 00:00:00 2001 From: Kaushik B Date: Fri, 22 Oct 2021 03:16:19 +0530 Subject: [PATCH 179/331] Fix sampler not being defined bug --- pytorch_lightning/lite/lite.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 8ca76ffbc7ce7..61d8714f28cbd 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -233,6 +233,7 @@ def _setup_dataloader( Returns: The wrapped dataloader. """ + sampler = dataloader.sampler if isinstance(dataloader, DataLoader): if replace_sampler and self._requires_distributed_sampler(dataloader): if not isinstance(dataloader.sampler, (SequentialSampler, RandomSampler)): From 2074c8b0ff0e8bc70a22048d3b45352beac2c782 Mon Sep 17 00:00:00 2001 From: Kaushik B Date: Fri, 22 Oct 2021 03:23:59 +0530 Subject: [PATCH 180/331] Add support for auto with the accelerator flag --- pytorch_lightning/lite/lite.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 61d8714f28cbd..49759930b5ac8 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -55,7 +55,7 @@ class LightningLite(ABC): - Multi-node support Args: - accelerator: The hardware to run on. Possible choices are: cpu, gpu, tpu. + accelerator: The hardware to run on. Possible choices are: cpu, gpu, tpu, auto. strategy: Strategy for how to run across multiple devices. Possible choices are: dp, ddp, ddp_spawn, tpu_spawn, deepspeed, ddp_sharded. devices: Number of devices to train on (int) or which GPUs to train on (list or str). The value applies @@ -233,10 +233,10 @@ def _setup_dataloader( Returns: The wrapped dataloader. """ - sampler = dataloader.sampler if isinstance(dataloader, DataLoader): + sampler = dataloader.sampler if replace_sampler and self._requires_distributed_sampler(dataloader): - if not isinstance(dataloader.sampler, (SequentialSampler, RandomSampler)): + if not isinstance(sampler, (SequentialSampler, RandomSampler)): raise MisconfigurationException( "You seem to have configured a sampler in your DataLoader. This will be replaced " " by `DistributedSampler` since `replace_sampler_ddp` is True and you are using" @@ -452,7 +452,7 @@ def _get_distributed_sampler(dataloader: DataLoader, **kwargs: Any) -> Distribut def _check_accelerator_support(self, accelerator: Optional[Union[str, Accelerator]]) -> None: if accelerator is None: return - supported = [t.lower() for t in self._supported_device_types()] + supported = [t.lower() for t in self._supported_device_types()] + ["auto"] if not isinstance(accelerator, (Accelerator, str)) or accelerator not in supported: raise MisconfigurationException( f"`accelerator={repr(accelerator)}` is not a valid choice." From 63fd03680b807c560918bd739fb811c852524bc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 21 Oct 2021 23:55:15 +0200 Subject: [PATCH 181/331] failing assert for strategy.model --- pytorch_lightning/lite/lite.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 49759930b5ac8..8dba310f3239f 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -266,17 +266,17 @@ def backward(self, tensor: Tensor, *args: Any, model: Optional[_LiteModule] = No When using ``strategy='deepspeed'`` and multiple models were setup, it is required to pass in the model as argument here. """ + module = model.module if model is not None else model if self._num_models > 0 and isinstance(self._strategy, DeepSpeedPlugin): - if not isinstance(model, _LiteModule): + if model is None: raise MisconfigurationException( "When using multiple models + deepspeed, please provide the model used to perform the optimization." ) # requires to attach the current `DeepSpeedEngine` for the `_LiteOptimizer.step` call. - self._strategy.model = model.module + self._strategy.model = module - assert self._strategy.model - self._precision_plugin._run_backward(tensor, self._strategy.model, *args, **kwargs) + self._precision_plugin._run_backward(tensor, module, *args, **kwargs) @contextmanager def cast(self) -> Generator[None, None, None]: From 240f95beeae222a0902471acffe00f1110322432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 22 Oct 2021 00:17:41 +0200 Subject: [PATCH 182/331] support Accelerator object and TrainingType strategy object to be passed to Lite --- pytorch_lightning/lite/lite.py | 12 +++++------- tests/lite/test_lite.py | 4 ++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 8dba310f3239f..810d728de01ac 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -450,20 +450,18 @@ def _get_distributed_sampler(dataloader: DataLoader, **kwargs: Any) -> Distribut return DistributedSampler(dataloader.dataset, **kwargs) def _check_accelerator_support(self, accelerator: Optional[Union[str, Accelerator]]) -> None: - if accelerator is None: - return - supported = [t.lower() for t in self._supported_device_types()] + ["auto"] - if not isinstance(accelerator, (Accelerator, str)) or accelerator not in supported: + supported = [t.value.lower() for t in self._supported_device_types()] + ["auto"] + valid = accelerator is None or isinstance(accelerator, Accelerator) or accelerator in supported + if not valid: raise MisconfigurationException( f"`accelerator={repr(accelerator)}` is not a valid choice." f" Choose one of {supported} or pass in a `Accelerator` instance." ) def _check_strategy_support(self, strategy: Optional[Union[str, TrainingTypePlugin]]) -> None: - if strategy is None: - return supported = [t.lower() for t in self._supported_strategy_types()] - if not isinstance(strategy, (TrainingTypePlugin, str)) and strategy not in supported: + valid = strategy is None or isinstance(strategy, TrainingTypePlugin) or strategy in supported + if not valid: raise MisconfigurationException( f"`strategy={repr(strategy)}` is not a valid choice." f" Choose one of {supported} or pass in a `TrainingTypePlugin` instance." diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index c9b0f39acc9f2..791f491a2d46c 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -52,13 +52,13 @@ def configure_optimizers(module: nn.Module): return torch.optim.SGD(module.parameters(), lr=0.0001) -@pytest.mark.parametrize("accelerator", ["coconut", Mock(spec=Accelerator)]) +@pytest.mark.parametrize("accelerator", ["coconut"]) def test_unsupported_accelerator(accelerator): with pytest.raises(MisconfigurationException, match=f"`accelerator={repr(accelerator)}` is not a valid choice"): EmptyLite(accelerator=accelerator) -@pytest.mark.parametrize("strategy", ["coconut", Mock(spec=TrainingTypePlugin)]) +@pytest.mark.parametrize("strategy", ["coconut"]) def test_unsupported_strategy(strategy): with pytest.raises(MisconfigurationException, match=f"`strategy={repr(strategy)}` is not a valid choice"): EmptyLite(strategy=strategy) From 56e3b7a2ebf3695ab2345629e82733d4b5fbe8d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 22 Oct 2021 00:36:42 +0200 Subject: [PATCH 183/331] support vararg optimizer sequence input to setup() --- pytorch_lightning/lite/lite.py | 34 +++++++++++----------------------- tests/lite/test_lite.py | 2 +- 2 files changed, 12 insertions(+), 24 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 810d728de01ac..abe0db8e671ae 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -154,23 +154,20 @@ def run(self, *args: Any, **kwargs: Any) -> Any: def setup( self, model: nn.Module, - optimizers: Union[Optimizer, List[Optimizer]], + *optimizers: Optimizer, move_to_device: bool = True, - ) -> Tuple[_LiteModule, Union[_LiteOptimizer, List[_LiteOptimizer]]]: + ) -> Tuple[Union[_LiteModule, _LiteOptimizer], ...]: """Setup a model and its optimizers for accelerated training. Args: model: A model to setup - optimizers: A list of optimizers to setup + *optimizers: One or multiple optimizers to setup move_to_device: If set ``True`` (default), moves the model to the correct device. Set this to ``False`` and alternatively use :meth:`to_device` manually. Returns: The tuple of the wrapped model and list of optimizers, in the same order they were passed in. """ - # wrap all objects passed in and return them in the same order - optimizers = [optimizers] if isinstance(optimizers, Optimizer) else optimizers - self._validate_setup(model, optimizers) if move_to_device: @@ -186,10 +183,12 @@ def setup( for param_group in optimizer.param_groups: param_group["params"] = [mapping.get(p, p) for p in param_group["params"]] - model, optimizers = self._setup_model_and_optimizers(model, optimizers) - optimizers = optimizers[0] if isinstance(optimizers, Sequence) and len(optimizers) == 1 else optimizers + # Let accelerator/plugin wrap and connect the models and optimizers + [model], optimizers = self._strategy._setup_models_and_optimizers([model], list(optimizers)) + model = _LiteModule(model, self._accelerator) + optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self._accelerator) for optimizer in optimizers] self._num_models += 1 - return model, optimizers + return model, *optimizers def setup_dataloaders( self, *dataloaders: DataLoader, replace_sampler: bool = True, move_to_device: bool = True @@ -426,17 +425,6 @@ def _set_deepspeed_precision_variables(self) -> None: precision = self._accelerator_connector.precision self._strategy.amp_level, self._strategy.amp_type, self._strategy._precision = amp_level, amp_type, precision - def _setup_model_and_optimizers( - self, - model: nn.Module, - optimizers: List[Optimizer], - ) -> Tuple[_LiteModule, Union[_LiteOptimizer, List[_LiteOptimizer]]]: - # Let accelerator/plugin wrap and connect the models and optimizers - [model], optimizers = self._strategy._setup_models_and_optimizers([model], optimizers) - model = _LiteModule(model, self._accelerator) - optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self._accelerator) for optimizer in optimizers] - return model, optimizers - def _requires_distributed_sampler(self, dataloader: DataLoader) -> bool: return ( self._accelerator_connector.is_distributed @@ -489,12 +477,12 @@ def _supported_strategy_types() -> Sequence[str]: ) @staticmethod - def _validate_setup(model: nn.Module, optimizers: List[Optimizer]) -> None: + def _validate_setup(model: nn.Module, optimizers: Sequence[Optimizer]) -> None: if isinstance(model, _LiteModule): - raise MisconfigurationException("A module should be passed only once to the ``setup`` method") + raise MisconfigurationException("A model should be passed only once to the `setup` method.") if any(isinstance(opt, _LiteOptimizer) for opt in optimizers): - raise MisconfigurationException("An optimizer should be passed only once to the ``setup`` method") + raise MisconfigurationException("An optimizer should be passed only once to the `setup` method.") @staticmethod def _validate_setup_dataloaders(*dataloaders: Union[DataLoader, List[DataLoader]]) -> None: diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 791f491a2d46c..ac39cc7f4c8de 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -210,7 +210,7 @@ def run(self, pass_model: bool = True): else: self.setup(model, optimizer_lite) - with pytest.raises(MisconfigurationException, match="A module should be passed only once to the"): + with pytest.raises(MisconfigurationException, match="A model should be passed only once to the"): runner = LiteRunner() runner.run() From a6cf010933bcebbb9d4d8b183b826f934b1e6291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 22 Oct 2021 00:51:37 +0200 Subject: [PATCH 184/331] remove redundant Iterable annotation from setup_dataloaders since we have an error message anyway --- pytorch_lightning/lite/lite.py | 13 +++++-------- tests/lite/test_lite.py | 14 +++++++------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index abe0db8e671ae..13b8a52d1c4a6 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -207,8 +207,7 @@ def setup_dataloaders( Returns: The wrapped dataloaders, in the same order they were passed in. """ - self._validate_setup_dataloaders(*dataloaders) - # user can call this method independently instead of the general purpose setup method + self._validate_setup_dataloaders(dataloaders) dataloaders = [ self._setup_dataloader(dataloader, replace_sampler=replace_sampler, move_to_device=move_to_device) for dataloader in dataloaders @@ -217,7 +216,7 @@ def setup_dataloaders( return dataloaders def _setup_dataloader( - self, dataloader: Union[Iterable, DataLoader], replace_sampler: bool = True, move_to_device: bool = True + self, dataloader: DataLoader, replace_sampler: bool = True, move_to_device: bool = True ) -> Union[Iterable, DataLoader]: """Setup a single dataloader for accelerated training. @@ -485,11 +484,9 @@ def _validate_setup(model: nn.Module, optimizers: Sequence[Optimizer]) -> None: raise MisconfigurationException("An optimizer should be passed only once to the `setup` method.") @staticmethod - def _validate_setup_dataloaders(*dataloaders: Union[DataLoader, List[DataLoader]]) -> None: + def _validate_setup_dataloaders(dataloaders: Sequence[DataLoader]) -> None: if any(isinstance(dl, _LiteDataLoader) for dl in dataloaders): - raise MisconfigurationException( - "A dataloader should be passed only once to the ``setup_dataloaders`` method" - ) + raise MisconfigurationException("A dataloader should be passed only once to the `setup_dataloaders` method") if any(not isinstance(dl, DataLoader) for dl in dataloaders): - raise MisconfigurationException("Only PyTorch DataLoader are currently supported.") + raise MisconfigurationException("Only PyTorch DataLoader are currently supported in `setup_dataloaders`.") diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index ac39cc7f4c8de..e761dc8fd2b3f 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -64,6 +64,13 @@ def test_unsupported_strategy(strategy): EmptyLite(strategy=strategy) +def test_setup_dataloaders_unsupported_type(): + """Test that the setup_dataloaders method fails when provided with non-DataLoader objects.""" + lite = EmptyLite() + with pytest.raises(MisconfigurationException, match="Only PyTorch DataLoader are currently supported"): + lite.setup_dataloaders(range(2)) + + def test_setup_dataloaders_return_type(): """Test that the setup method returns the dataloaders wrapped as LiteDataLoader and in the right order.""" lite = EmptyLite() @@ -84,13 +91,6 @@ def test_setup_dataloaders_return_type(): assert lite_dataloader1.dataset is dataset1 -def test_lite_with_iterable(): - """Test that the setup_dataloaders method fails when provided with an iterable.""" - lite = EmptyLite() - with pytest.raises(MisconfigurationException, match="Only PyTorch DataLoader are currently supported"): - lite.setup_dataloaders(range(2)) - - @mock.patch( "pytorch_lightning.lite.lite.LightningLite.device", new_callable=PropertyMock, From 070fa235af87d67ff8af783a0ec5f9e4dcad8380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 22 Oct 2021 01:05:49 +0200 Subject: [PATCH 185/331] to_device overload for mypy --- pytorch_lightning/lite/lite.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 13b8a52d1c4a6..e2cfcb13687ca 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -17,7 +17,7 @@ from contextlib import contextmanager from functools import partial from pathlib import Path -from typing import Any, Dict, Generator, Iterable, List, Optional, Sequence, Tuple, Union +from typing import Any, Dict, Generator, Iterable, List, Optional, Sequence, Tuple, Union, overload import torch import torch.nn as nn @@ -286,6 +286,18 @@ def cast(self) -> Generator[None, None, None]: with self._precision_plugin.forward_context(): yield + @overload + def to_device(self, obj: nn.Module) -> nn.Module: + pass + + @overload + def to_device(self, obj: Tensor) -> Tensor: + pass + + @overload + def to_device(self, obj: Any) -> Any: + pass + def to_device(self, obj: Union[nn.Module, Tensor, Any]) -> Union[nn.Module, Tensor, Any]: """Move a :class:`torch.nn.Module` or a collection of tensors to the current device, if it is not already on that device. From 6aa4ac0923d75780614caa70832c511529df5037 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 21 Oct 2021 23:07:17 +0000 Subject: [PATCH 186/331] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pytorch_lightning/lite/lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index e2cfcb13687ca..10246deb287c0 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -17,7 +17,7 @@ from contextlib import contextmanager from functools import partial from pathlib import Path -from typing import Any, Dict, Generator, Iterable, List, Optional, Sequence, Tuple, Union, overload +from typing import Any, Dict, Generator, Iterable, List, Optional, overload, Sequence, Tuple, Union import torch import torch.nn as nn From 0838ea99803ea4bd799942fc93d6391005725c7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 22 Oct 2021 01:45:31 +0200 Subject: [PATCH 187/331] harden tests for setup() --- pytorch_lightning/lite/lite.py | 8 ++-- tests/lite/test_lite.py | 68 +++++++++++++++++++++++----------- 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 10246deb287c0..6ebf77287a1ca 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -156,12 +156,12 @@ def setup( model: nn.Module, *optimizers: Optimizer, move_to_device: bool = True, - ) -> Tuple[Union[_LiteModule, _LiteOptimizer], ...]: + ) -> Union[_LiteModule, Tuple[Union[_LiteModule, _LiteOptimizer], ...]]: """Setup a model and its optimizers for accelerated training. Args: model: A model to setup - *optimizers: One or multiple optimizers to setup + *optimizers: The optimizer(s) to setup (no optimizers is also possible) move_to_device: If set ``True`` (default), moves the model to the correct device. Set this to ``False`` and alternatively use :meth:`to_device` manually. @@ -188,7 +188,9 @@ def setup( model = _LiteModule(model, self._accelerator) optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self._accelerator) for optimizer in optimizers] self._num_models += 1 - return model, *optimizers + if optimizers: + return model, *optimizers + return model def setup_dataloaders( self, *dataloaders: DataLoader, replace_sampler: bool = True, move_to_device: bool = True diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index e761dc8fd2b3f..dfb93fc1a8a31 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -26,7 +26,7 @@ from pytorch_lightning import seed_everything from pytorch_lightning.accelerators import Accelerator from pytorch_lightning.lite import LightningLite -from pytorch_lightning.lite.wrappers import _LiteDataLoader +from pytorch_lightning.lite.wrappers import _LiteDataLoader, _LiteModule, _LiteOptimizer from pytorch_lightning.plugins import DeepSpeedPlugin, PrecisionPlugin, TrainingTypePlugin from pytorch_lightning.utilities.exceptions import MisconfigurationException from tests.helpers.boring_model import RandomDataset @@ -64,11 +64,55 @@ def test_unsupported_strategy(strategy): EmptyLite(strategy=strategy) +def test_setup_optimizers(): + """Test that setup_optimizers can handle no optimizers, one optimizer, or multiple optimizers.""" + lite = EmptyLite() + model = nn.Linear(1, 2) + optimizer0 = torch.optim.SGD(model.parameters(), lr=0.1) + optimizer1 = torch.optim.Adam(model.parameters(), lr=0.1) + + # no optimizer + lite_model = lite.setup(model) + assert isinstance(lite_model, _LiteModule) + assert lite_model.module is model + + # single optimizer + lite_model, lite_optimizer = lite.setup(model, optimizer0) + assert isinstance(lite_model, _LiteModule) + assert isinstance(lite_optimizer, _LiteOptimizer) + assert lite_model.module is model + assert lite_optimizer.optimizer is optimizer0 + + # multiple optimizers + lite_model, lite_optimizer0, lite_optimizer1 = lite.setup(model, optimizer0, optimizer1) + assert isinstance(lite_model, _LiteModule) + assert isinstance(lite_optimizer0, _LiteOptimizer) + assert isinstance(lite_optimizer1, _LiteOptimizer) + assert lite_model.module is model + assert lite_optimizer0.optimizer is optimizer0 + assert lite_optimizer1.optimizer is optimizer1 + + +def test_setup_twice_fails(): + """Test that calling setup with a model or optimizer that is already wrapped fails.""" + lite = EmptyLite() + model = nn.Linear(1, 2) + optimizer = torch.optim.Adam(model.parameters()) + + lite_model, lite_optimizer = lite.setup(model, optimizer) + with pytest.raises(MisconfigurationException, match="A model should be passed only once to the"): + lite.setup(lite_model, optimizer) + + lite_model, lite_optimizer = lite.setup(model, optimizer) + with pytest.raises(MisconfigurationException, match="An optimizer should be passed only once to the"): + lite.setup(model, lite_optimizer) + + def test_setup_dataloaders_unsupported_type(): """Test that the setup_dataloaders method fails when provided with non-DataLoader objects.""" lite = EmptyLite() with pytest.raises(MisconfigurationException, match="Only PyTorch DataLoader are currently supported"): - lite.setup_dataloaders(range(2)) + lite.setup_dataloaders(range(2)) # type: ignore def test_setup_dataloaders_return_type(): @@ -199,26 +243,6 @@ def test_backward(): lite._precision_plugin._run_backward.assert_called_with(loss, None, "arg", keyword="kwarg") -def test_lightning_lite_setup(): - class LiteRunner(LightningLite): - def run(self, pass_model: bool = True): - model = BoringModel() - optimizer = configure_optimizers(model) - model_lite, optimizer_lite = self.setup(model, optimizer) - if pass_model: - self.setup(model_lite, optimizer) - else: - self.setup(model, optimizer_lite) - - with pytest.raises(MisconfigurationException, match="A model should be passed only once to the"): - runner = LiteRunner() - runner.run() - - with pytest.raises(MisconfigurationException, match="An optimizer should be passed only once to the"): - runner = LiteRunner() - runner.run(pass_model=False) - - def test_lightning_lite_setup_dataloaders(): class LiteRunner(LightningLite): def run(self): From e6e28957dc968a51370d3e2e7b665f93670491f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 22 Oct 2021 01:49:51 +0200 Subject: [PATCH 188/331] simplify test --- tests/lite/test_lite.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index dfb93fc1a8a31..a511bb8f29805 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -135,6 +135,16 @@ def test_setup_dataloaders_return_type(): assert lite_dataloader1.dataset is dataset1 +def test_setup_dataloaders_twice_fails(): + """Test that calling setup_dataloaders with a dataloader that is already wrapped fails.""" + lite = EmptyLite() + dataloader = DataLoader(range(2)) + lite_dataloader = lite.setup_dataloaders(dataloader) + + with pytest.raises(MisconfigurationException, match="A dataloader should be passed only once to the"): + lite.setup_dataloaders(lite_dataloader) + + @mock.patch( "pytorch_lightning.lite.lite.LightningLite.device", new_callable=PropertyMock, @@ -243,19 +253,6 @@ def test_backward(): lite._precision_plugin._run_backward.assert_called_with(loss, None, "arg", keyword="kwarg") -def test_lightning_lite_setup_dataloaders(): - class LiteRunner(LightningLite): - def run(self): - - dataloader = DataLoader(RandomDataset(32, 64)) - dataloader_lite = self.setup_dataloaders(dataloader) - _ = self.setup_dataloaders(dataloader_lite) - - with pytest.raises(MisconfigurationException, match="A dataloader should be passed only once to the"): - runner = LiteRunner() - runner.run() - - def test_lightning_lite_track_model_setup(): class LiteRunner(LightningLite): def run(self): From 314da4a6e2ef1b5747e0fa923d222b925a32ce3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 22 Oct 2021 02:05:33 +0200 Subject: [PATCH 189/331] fix mypy for setup() return type --- pytorch_lightning/lite/lite.py | 4 ++-- tests/lite/test_lite.py | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 6ebf77287a1ca..81ede937c5280 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -156,7 +156,7 @@ def setup( model: nn.Module, *optimizers: Optimizer, move_to_device: bool = True, - ) -> Union[_LiteModule, Tuple[Union[_LiteModule, _LiteOptimizer], ...]]: + ) -> Union[_LiteModule, List[Union[_LiteModule, _LiteOptimizer]]]: """Setup a model and its optimizers for accelerated training. Args: @@ -189,7 +189,7 @@ def setup( optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self._accelerator) for optimizer in optimizers] self._num_models += 1 if optimizers: - return model, *optimizers + return [model] + optimizers # type: ignore return model def setup_dataloaders( diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index a511bb8f29805..9f23e441cbca7 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -24,12 +24,10 @@ from torch.utils.data import DataLoader, DistributedSampler, Sampler from pytorch_lightning import seed_everything -from pytorch_lightning.accelerators import Accelerator from pytorch_lightning.lite import LightningLite from pytorch_lightning.lite.wrappers import _LiteDataLoader, _LiteModule, _LiteOptimizer from pytorch_lightning.plugins import DeepSpeedPlugin, PrecisionPlugin, TrainingTypePlugin from pytorch_lightning.utilities.exceptions import MisconfigurationException -from tests.helpers.boring_model import RandomDataset from tests.helpers.runif import RunIf From 5efbfb3c3a500f26268d30f2a684690d3208d0bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 22 Oct 2021 02:06:23 +0200 Subject: [PATCH 190/331] organize --- pytorch_lightning/lite/lite.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 81ede937c5280..ff4bd73910956 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -103,12 +103,11 @@ def __init__( self._accelerator = self._accelerator_connector.accelerator self._strategy = self._accelerator.training_type_plugin self._precision_plugin = self._accelerator.precision_plugin + self._num_models: int = 0 # wrap the run method so we can inject setup logic or spawn processes for the user setattr(self, "run", self._run_wrapper(self.run)) - self._num_models: int = 0 - @property def device(self) -> torch.device: """The current device this process runs on. From 676f765a3ae0bc47f10854a1857c48b01a7828ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 22 Oct 2021 02:17:17 +0200 Subject: [PATCH 191/331] remove dataloader type check (already checked above) --- pytorch_lightning/lite/lite.py | 35 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index ff4bd73910956..3d0ca3100267a 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -232,24 +232,23 @@ def _setup_dataloader( Returns: The wrapped dataloader. """ - if isinstance(dataloader, DataLoader): - sampler = dataloader.sampler - if replace_sampler and self._requires_distributed_sampler(dataloader): - if not isinstance(sampler, (SequentialSampler, RandomSampler)): - raise MisconfigurationException( - "You seem to have configured a sampler in your DataLoader. This will be replaced " - " by `DistributedSampler` since `replace_sampler_ddp` is True and you are using" - " distributed training. Either remove the sampler from your DataLoader or set" - " `replace_sampler=False` if you want to use your custom sampler." - ) - sampler = self._get_distributed_sampler(dataloader, **self._strategy.distributed_sampler_kwargs) - - kwargs = TrainerDataLoadingMixin._get_dataloader_init_kwargs(dataloader, sampler) - device = self.device if move_to_device else None - if isinstance(self._strategy, TPUSpawnPlugin): - dataloader = DataLoader(**kwargs) - else: - dataloader = _LiteDataLoader(device=device, **kwargs) + sampler = dataloader.sampler + if replace_sampler and self._requires_distributed_sampler(dataloader): + if not isinstance(sampler, (SequentialSampler, RandomSampler)): + raise MisconfigurationException( + "You seem to have configured a sampler in your DataLoader. This will be replaced " + " by `DistributedSampler` since `replace_sampler_ddp` is True and you are using" + " distributed training. Either remove the sampler from your DataLoader or set" + " `replace_sampler=False` if you want to use your custom sampler." + ) + sampler = self._get_distributed_sampler(dataloader, **self._strategy.distributed_sampler_kwargs) + + kwargs = TrainerDataLoadingMixin._get_dataloader_init_kwargs(dataloader, sampler) + device = self.device if move_to_device else None + if isinstance(self._strategy, TPUSpawnPlugin): + dataloader = DataLoader(**kwargs) + else: + dataloader = _LiteDataLoader(device=device, **kwargs) return self._strategy.process_dataloader(dataloader) def backward(self, tensor: Tensor, *args: Any, model: Optional[_LiteModule] = None, **kwargs: Any) -> None: From 04e1b417c8c9728c071ce1a8efd66a86ed3c6b8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 22 Oct 2021 02:30:33 +0200 Subject: [PATCH 192/331] update examples, setup() syntax --- pl_examples/lite_examples/pytorch_2_lite_2_lightning.py | 2 +- tests/lite/test_parity.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pl_examples/lite_examples/pytorch_2_lite_2_lightning.py b/pl_examples/lite_examples/pytorch_2_lite_2_lightning.py index 1c81907816597..592d5a7ab951b 100644 --- a/pl_examples/lite_examples/pytorch_2_lite_2_lightning.py +++ b/pl_examples/lite_examples/pytorch_2_lite_2_lightning.py @@ -132,7 +132,7 @@ def run(self, model: nn.Module, train_dataloader: DataLoader, val_dataloader: Da # You would need to call `self.setup` to wrap `model` and `optimizer`. If you # # have multiple models (c.f GAN), call `setup` for each one of them and their # # associated optimizers. # - model, optimizer = self.setup(model=model, optimizers=optimizer) + model, optimizer = self.setup(model, optimizer) ################################################################################### ################################################################################### diff --git a/tests/lite/test_parity.py b/tests/lite/test_parity.py index 423a606561d2d..4b52448ceff71 100644 --- a/tests/lite/test_parity.py +++ b/tests/lite/test_parity.py @@ -77,7 +77,7 @@ def main( class LiteRunner(LightningLite): def run(self, model: nn.Module, train_dataloader: DataLoader, num_epochs: int = 10, tmpdir: str = None): optimizer = configure_optimizers(model) - model, optimizer = self.setup(model=model, optimizers=optimizer) + model, optimizer = self.setup(model, optimizer) train_dataloader = self.setup_dataloaders(train_dataloader) model.train() From 024fa6ac6650c92cb891b4a88a9c4eadfd57860a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 22 Oct 2021 02:49:59 +0200 Subject: [PATCH 193/331] skip test if dependency not available --- pytorch_lightning/lite/lite.py | 1 - tests/lite/test_lite.py | 27 +++++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 3d0ca3100267a..dda45d505b22a 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -481,7 +481,6 @@ def _supported_strategy_types() -> Sequence[str]: DistributedType.DDP, DistributedType.DDP_SPAWN, DistributedType.TPU_SPAWN, - DistributedType.DP, DistributedType.DEEPSPEED, DistributedType.DDP_SHARDED, DistributedType.DDP_SHARDED_SPAWN, diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 9f23e441cbca7..e31be6de16db8 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -29,6 +29,7 @@ from pytorch_lightning.plugins import DeepSpeedPlugin, PrecisionPlugin, TrainingTypePlugin from pytorch_lightning.utilities.exceptions import MisconfigurationException from tests.helpers.runif import RunIf +from pytorch_lightning.utilities import DistributedType class EmptyLite(LightningLite): @@ -172,7 +173,18 @@ def test_setup_dataloaders_distributed_sampler_not_needed(): assert lite_dataloader.sampler is custom_sampler -@pytest.mark.parametrize("strategy", LightningLite._supported_strategy_types()) +@pytest.mark.parametrize( + "strategy", + [ + DistributedType.DP, + DistributedType.DDP, + DistributedType.DDP_SPAWN, + DistributedType.TPU_SPAWN, + pytest.param(DistributedType.DEEPSPEED, marks=RunIf(deepspeed=True)), + pytest.param(DistributedType.DDP_SHARDED, marks=RunIf(fairscale=True)), + pytest.param(DistributedType.DDP_SHARDED_SPAWN, marks=RunIf(fairscale=True)), + ], +) def test_setup_dataloaders_replace_custom_sampler(strategy): """Test that asking to replace a custom sampler results in an error when a distributed sampler would be needed.""" @@ -190,7 +202,18 @@ def test_setup_dataloaders_replace_custom_sampler(strategy): assert lite_dataloader.sampler is custom_sampler -@pytest.mark.parametrize("strategy", LightningLite._supported_strategy_types()) +@pytest.mark.parametrize( + "strategy", + [ + DistributedType.DP, + DistributedType.DDP, + DistributedType.DDP_SPAWN, + DistributedType.TPU_SPAWN, + pytest.param(DistributedType.DEEPSPEED, marks=RunIf(deepspeed=True)), + pytest.param(DistributedType.DDP_SHARDED, marks=RunIf(fairscale=True)), + pytest.param(DistributedType.DDP_SHARDED_SPAWN, marks=RunIf(fairscale=True)), + ], +) @pytest.mark.parametrize("shuffle", [True, False]) def test_setup_dataloaders_replace_standard_sampler(shuffle, strategy): """Test that Lite replaces the default samplers with DistributedSampler automatically.""" From 3e446d94bb4872c2f7327c5786f2eb73e2289cdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 22 Oct 2021 02:53:32 +0200 Subject: [PATCH 194/331] skip test if deepspeed unavailable --- tests/lite/test_lite.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index e31be6de16db8..20f6b545fb537 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -294,6 +294,7 @@ def run(self): # TODO: This test does not assert any functionality: use Mock to assert how DeepSpeedPlugin gets called +@RunIf(deepspeed=True) @mock.patch("pytorch_lightning.plugins.DeepSpeedPlugin.setup_distributed", lambda x: x) def test_lightning_lite_deepspeed_backward(): class LiteRunner(LightningLite): From 18c58fdfd59a22afb5dc8bfc18100b2e92626e7f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 22 Oct 2021 00:54:48 +0000 Subject: [PATCH 195/331] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/lite/test_lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 20f6b545fb537..9620dc771f005 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -27,9 +27,9 @@ from pytorch_lightning.lite import LightningLite from pytorch_lightning.lite.wrappers import _LiteDataLoader, _LiteModule, _LiteOptimizer from pytorch_lightning.plugins import DeepSpeedPlugin, PrecisionPlugin, TrainingTypePlugin +from pytorch_lightning.utilities import DistributedType from pytorch_lightning.utilities.exceptions import MisconfigurationException from tests.helpers.runif import RunIf -from pytorch_lightning.utilities import DistributedType class EmptyLite(LightningLite): From 186ed8da075b5f39ceb337dca8c5a59f8299069e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 22 Oct 2021 04:17:56 +0200 Subject: [PATCH 196/331] test run() input outputs --- tests/lite/test_lite.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 9620dc771f005..4a132af797784 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -21,6 +21,7 @@ import torch.distributed import torch.nn.functional from torch import nn +from torch.optim import Optimizer from torch.utils.data import DataLoader, DistributedSampler, Sampler from pytorch_lightning import seed_everything @@ -63,6 +64,26 @@ def test_unsupported_strategy(strategy): EmptyLite(strategy=strategy) +def test_run_input_output(): + """Test that the dynamically patched run() method receives the input arguments and returns the result.""" + + class Lite(LightningLite): + + run_args = () + run_kwargs = {} + + def run(self, *args, **kwargs): + self.run_args = args + self.run_kwargs = kwargs + return "result" + + lite = Lite() + result = lite.run(1, 2, three=3) + assert result == "result" + assert lite.run_args == (1, 2) + assert lite.run_kwargs == {"three": 3} + + def test_setup_optimizers(): """Test that setup_optimizers can handle no optimizers, one optimizer, or multiple optimizers.""" lite = EmptyLite() From 442a1847876bde7feac7057799c4c716f06df05d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 22 Oct 2021 04:18:16 +0200 Subject: [PATCH 197/331] re-organize tests --- tests/lite/test_lite.py | 64 +++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 37 deletions(-) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 4a132af797784..9c5238517d096 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -128,6 +128,20 @@ def test_setup_twice_fails(): lite.setup(model, lite_optimizer) +def test_setup_tracks_num_models(): + """Test that setup() tracks how many times it has setup a model.""" + lite = EmptyLite() + model = nn.Linear(1, 2) + optimizer = torch.optim.Adam(model.parameters()) + + assert lite._num_models == 0 + lite.setup(model, optimizer) + assert lite._num_models == 1 + + lite.setup(model, optimizer) + assert lite._num_models == 2 + + def test_setup_dataloaders_unsupported_type(): """Test that the setup_dataloaders method fails when provided with non-DataLoader objects.""" lite = EmptyLite() @@ -295,50 +309,26 @@ def test_backward(): lite._precision_plugin._run_backward.assert_called_with(loss, None, "arg", keyword="kwarg") -def test_lightning_lite_track_model_setup(): - class LiteRunner(LightningLite): - def run(self): - model = BoringModel() - optimizer = configure_optimizers(model) - - assert self._num_models == 0 - self.setup(model, optimizer) - assert self._num_models == 1 - - model = BoringModel() - optimizer = configure_optimizers(model) - self.setup(model, optimizer) - assert self._num_models == 2 - - runner = LiteRunner() - runner.run() +@RunIf(deepspeed=True) +def test_backward_model_input_required(): + """Test that when using deepspeed and multiple models, backward() requires the model as input.""" + lite = EmptyLite(strategy="deepspeed") + model0 = nn.Linear(1, 2) + model1 = nn.Linear(1, 2) -# TODO: This test does not assert any functionality: use Mock to assert how DeepSpeedPlugin gets called -@RunIf(deepspeed=True) -@mock.patch("pytorch_lightning.plugins.DeepSpeedPlugin.setup_distributed", lambda x: x) -def test_lightning_lite_deepspeed_backward(): - class LiteRunner(LightningLite): - def run(self): - def fn(*args): - return args + optimizer0 = torch.optim.Adam(model0.parameters()) + optimizer1 = torch.optim.Adam(model1.parameters()) - self._strategy._setup_model_and_optimizer = fn - model = BoringModel() - optimizer = configure_optimizers(model) - self.setup(model, optimizer) + lite._strategy._setup_model_and_optimizer = lambda *args: args - model = BoringModel() - optimizer = configure_optimizers(model) - self.setup(model, optimizer) + lite.setup(model0, optimizer0) + lite.setup(model1, optimizer1) - x = model(torch.randn(1, 32)) - loss = x.sum() - self.backward(loss) + loss = model0(torch.randn(1, 1)).sum() with pytest.raises(MisconfigurationException, match="please provide the model used to perform"): - runner = LiteRunner(strategy="deepspeed") - runner.run() + lite.backward(loss) @RunIf(min_gpus=2, deepspeed=True, special=True) From 3539f2cfa0f635b193ec94a858976be0c4a7bd60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 22 Oct 2021 04:20:15 +0200 Subject: [PATCH 198/331] rename --- tests/lite/test_lite.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 9c5238517d096..eb21b71177121 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -48,10 +48,6 @@ def forward(self, x): return torch.nn.functional.mse_loss(x, torch.ones_like(x)) -def configure_optimizers(module: nn.Module): - return torch.optim.SGD(module.parameters(), lr=0.0001) - - @pytest.mark.parametrize("accelerator", ["coconut"]) def test_unsupported_accelerator(accelerator): with pytest.raises(MisconfigurationException, match=f"`accelerator={repr(accelerator)}` is not a valid choice"): @@ -333,10 +329,10 @@ def test_backward_model_input_required(): @RunIf(min_gpus=2, deepspeed=True, special=True) def test_deepspeed_multiple_models(): - class LiteRunner(LightningLite): + class Lite(LightningLite): def run(self): model = BoringModel() - optimizer = configure_optimizers(model) + optimizer = torch.optim.SGD(model.parameters(), lr=0.0001) model, optimizer = self.setup(model, optimizer) state_dict = deepcopy(model.state_dict()) @@ -352,11 +348,11 @@ def run(self): seed_everything(42) model_1 = BoringModel() - optimizer_1 = configure_optimizers(model_1) + optimizer_1 = torch.optim.SGD(model_1.parameters(), lr=0.0001) seed_everything(42) model_2 = BoringModel() - optimizer_2 = configure_optimizers(model_2) + optimizer_2 = torch.optim.SGD(model_2.parameters(), lr=0.0001) for mw_1, mw_2 in zip(model_1.state_dict().values(), model_2.state_dict().values()): assert torch.equal(mw_1, mw_2) @@ -394,4 +390,4 @@ def run(self): assert self.broadcast(True) assert self.is_global_zero == (self.local_rank == 0) - LiteRunner(strategy=DeepSpeedPlugin(stage=3, logging_batch_size_per_gpu=1), devices=2, accelerator="gpu").run() + Lite(strategy=DeepSpeedPlugin(stage=3, logging_batch_size_per_gpu=1), devices=2, accelerator="gpu").run() From 1510661d4d857bb1e262d7a2b0c51b8aa6c32c32 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Fri, 22 Oct 2021 12:39:57 +0100 Subject: [PATCH 199/331] Add LightningLite documentation (#10043) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrian Wälchli Co-authored-by: Rohit Gupta --- CHANGELOG.md | 2 +- docs/source/api_references.rst | 11 + docs/source/conf.py | 10 +- docs/source/index.rst | 2 +- docs/source/starter/lightning_lite.rst | 284 +++++++++++++++++++++++++ pytorch_lightning/lite/lite.py | 2 +- 6 files changed, 305 insertions(+), 6 deletions(-) create mode 100644 docs/source/starter/lightning_lite.rst diff --git a/CHANGELOG.md b/CHANGELOG.md index 655484292ee59..cf807807225ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -218,7 +218,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). * Implemented `DeepSpeedPlugin._setup_models_and_optimizers` ([#10009](https://github.com/PyTorchLightning/pytorch-lightning/pull/10009)) * Implemented `{DDPShardedPlugin,DDPShardedSpawnPlugin}._setup_models_and_optimizers` ([#10028](https://github.com/PyTorchLightning/pytorch-lightning/pull/10028)) * Added optional `model` argument to the `optimizer_step` methods in accelerators and plugins ([#10023](https://github.com/PyTorchLightning/pytorch-lightning/pull/10023)) - + * Add `LightningLite` documentation ([#10043](https://github.com/PyTorchLightning/pytorch-lightning/pull/10043)) - Added `XLACheckpointIO` plugin ([#9972](https://github.com/PyTorchLightning/pytorch-lightning/pull/9972)) diff --git a/docs/source/api_references.rst b/docs/source/api_references.rst index b2d546a158f42..ca3220b0f5592 100644 --- a/docs/source/api_references.rst +++ b/docs/source/api_references.rst @@ -243,6 +243,17 @@ Trainer API trainer +LightningLite API +----------------- + +.. currentmodule:: pytorch_lightning.lite + +.. autosummary:: + :toctree: api + :nosignatures: + + LightningLite + Tuner API --------- diff --git a/docs/source/conf.py b/docs/source/conf.py index f5f9605263217..cbd7a51fa1238 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -16,6 +16,7 @@ import os import shutil import sys +import warnings from importlib.util import module_from_spec, spec_from_file_location import pt_lightning_sphinx_theme @@ -26,10 +27,13 @@ sys.path.insert(0, os.path.abspath(PATH_ROOT)) sys.path.append(os.path.join(PATH_RAW_NB, ".actions")) +_SHOULD_COPY_NOTEBOOKS = True + try: from helpers import HelperCLI except Exception: - raise ModuleNotFoundError("To build the code, please run: `git submodule update --init --recursive`") + _SHOULD_COPY_NOTEBOOKS = False + warnings.warn("To build the code, please run: `git submodule update --init --recursive`", stacklevel=2) FOLDER_GENERATED = "generated" SPHINX_MOCK_REQUIREMENTS = int(os.environ.get("SPHINX_MOCK_REQUIREMENTS", True)) @@ -41,8 +45,8 @@ spec.loader.exec_module(about) # -- Project documents ------------------------------------------------------- - -HelperCLI.copy_notebooks(PATH_RAW_NB, PATH_HERE, "notebooks") +if _SHOULD_COPY_NOTEBOOKS: + HelperCLI.copy_notebooks(PATH_RAW_NB, PATH_HERE, "notebooks") def _transform_changelog(path_in: str, path_out: str) -> None: diff --git a/docs/source/index.rst b/docs/source/index.rst index ea3e606d72849..ecb0c1c39dd78 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -21,6 +21,7 @@ PyTorch Lightning starter/new-project starter/converting starter/rapid_prototyping_templates + starter/lightning_lite .. toctree:: :maxdepth: 1 @@ -33,7 +34,6 @@ PyTorch Lightning Lightning project template benchmarking/benchmarks - .. toctree:: :maxdepth: 2 :name: pl_docs diff --git a/docs/source/starter/lightning_lite.rst b/docs/source/starter/lightning_lite.rst new file mode 100644 index 0000000000000..dead2f9efdd35 --- /dev/null +++ b/docs/source/starter/lightning_lite.rst @@ -0,0 +1,284 @@ +########################################### +LightningLite - Stepping Stone to Lightning +########################################### + + +.. image:: https://pl-public-data.s3.amazonaws.com/docs/static/images/lite/lightning_lite.gif + :alt: Animation showing how to convert a standard training loop to a Lightning loop + + + +:class:`~pytorch_lightning.lite.LightningLite` enables pure PyTorch users to scale their existing code +on any kind of device while retaining full control over their own loops and optimization logic. + +:class:`~pytorch_lightning.lite.LightningLite` is the right tool for you if you match one of the two following descriptions: + +- I want to quickly scale my existing code to multiple devices with minimal code changes. + +- I would like to convert my existing code to the Lightning API, but a full path to Lightning transition might be too complex. I am looking for a stepping stone to ensure reproducibility during the transition. + +Supported Integrations +====================== + +:class:`~pytorch_lightning.lite.LightningLite` supports single and multiple models and optimizers. + +.. list-table:: + :widths: 50 50 + :header-rows: 1 + + * - LightningLite arguments + - Possible choices + * - ``accelerator`` + - ``cpu``, ``gpu``, ``tpu``, ``auto`` + * - ``strategy`` + - ``dp``, ``ddp``, ``ddp_spawn``, ``ddp_sharded``, ``ddp_sharded_spawn``, ``deepspeed`` + * - ``precision`` + - ``16``, ``bf16``, ``32``, ``64`` + * - ``clusters`` + - ``TorchElastic``, ``SLURM``, ``Kubeflow``, ``LSF`` + + +Coming soon: IPU accelerator, support for Horovod as a strategy and fully sharded training. + + +################ +Learn by example +################ + +My existing PyTorch code +======================== + +The ``run`` function contains custom training loop used to train ``MyModel`` on ``MyDataset`` for ``num_epochs`` epochs. + +.. code-block:: python + + import torch + from torch import nn + from torch.utils.data import DataLoader, Dataset + + + class MyModel(nn.Module): + ... + + + class MyDataset(Dataset): + ... + + + def run(num_epochs: int): + + device = "cuda" if torch.cuda.is_available() else "cpu" + + model = MyModel(...).to(device) + optimizer = torch.optim.SGD(model.parameters(), ...) + + dataloader = DataLoader(MyDataset(...), ...) + + model.train() + for epoch in range(num_epochs): + for batch in dataloader: + batch = batch.to(device) + optimizer.zero_grad() + loss = model(batch) + loss.backward() + optimizer.step() + + + run(10) + +Convert to LightningLite +======================== + +Here are 4 required steps to convert to :class:`~pytorch_lightning.lite.LightningLite`. + +1. Subclass :class:`~pytorch_lightning.lite.LightningLite` and override its :meth:`~pytorch_lightning.lite.LightningLite.run` method. +2. Move the body of your existing `run` function. +3. Apply :meth:`~pytorch_lightning.lite.LightningLite.setup` over each model and optimizers pair, :meth:`~pytorch_lightning.lite.LightningLite.setup_dataloaders` on all your dataloaders and replace ``loss.backward()`` by ``self.backward(loss)`` +4. Instantiate your :class:`~pytorch_lightning.lite.LightningLite` and call its :meth:`~pytorch_lightning.lite.LightningLite.run` method. + + +.. code-block:: python + + import torch + from torch import nn + from torch.utils.data import DataLoader, Dataset + from pytorch_lightning.lite import LightningLite + + + class MyModel(nn.Module): + ... + + + class MyDataset(Dataset): + ... + + + class Lite(LightningLite): + def run(self, num_epochs: int): + + model = MyModel(...) + optimizer = torch.optim.SGD(model.parameters(), ...) + + model, optimizer = self.setup(model, optimizer) + + dataloader = DataLoader(MyDataset(...), ...) + dataloader = self.setup_dataloaders(dataloader) + + model.train() + for epoch in range(num_epochs): + for batch in dataloader: + optimizer.zero_grad() + loss = model(batch) + self.backward(loss) + optimizer.step() + + + Lite(...).run(10) + + +That's all. You can now train on any kind of device and scale your training. + +The :class:`~pytorch_lightning.lite.LightningLite` takes care of device management, so you don't have to. + +You can remove any device specific logic within your code. + +Here is how to train on 8 GPUs with `torch.bfloat16 `_ precision: + +.. code-block:: python + + Lite(strategy="ddp", devices=8, accelerator="gpu", precision="bf16").run(10) + +Here is how to use `DeepSpeed Zero3 `_ with 8 GPUs and precision 16: + +.. code-block:: python + + Lite(strategy="deepspeed", devices=8, accelerator="gpu", precision=16).run(10) + +Lightning can also figure it automatically for you ! + +.. code-block:: python + + Lite(devices="auto", accelerator="auto", precision=16).run(10) + + +You can also easily use distributed collectives if required. +Here is an example while running on 256 GPUs. + +.. code-block:: python + + class Lite(LightningLite): + def run(self): + + # Transfer and concatenate tensors across processes + self.all_gather(...) + + # Transfer an object from one process to all the others + self.broadcast(..., src=...) + + # The total number of processes running across all devices and nodes. + self.world_size + + # The global index of the current process across all devices and nodes. + self.global_rank + + # The index of the current process among the processes running on the local node. + self.local_rank + + # The index of the current node. + self.node_rank + + # Wether this global rank is rank zero. + if self.is_global_zero: + # do something on rank 0 + ... + + # Wait for all processes to enter this call. + self.barrier() + + # Reduce a boolean decision across processes. + self.reduce_decision(...) + + + Lite(strategy="ddp", gpus=8, num_nodes=32, accelerator="gpu").run() + + +.. note:: We recommend instantiating the models within the :meth:`~pytorch_lightning.lite.LightningLite.run` method as large models would cause OOM Error otherwise. + + +Distributed Training Pitfalls +============================= + +The :class:`~pytorch_lightning.lite.LightningLite` provides you only with the tool to scale your training, +but there are several major challenges ahead of you now: + + +.. list-table:: + :widths: 50 50 + :header-rows: 0 + + * - Processes divergence + - This happens when processes execute different section of the code due to different if/else condition, race condition on existing files, etc., resulting in hanging. + * - Cross processes reduction + - Wrongly reported metrics or gradients due mis-reduction. + * - Large sharded models + - Instantiation, materialization and state management of large models. + * - Rank 0 only actions + - Logging, profiling, etc. + * - Checkpointing / Early stopping / Callbacks + - Ability to easily customize your training behaviour and make it stateful. + * - Batch-level fault tolerance training + - Ability to resume from a failure as if it never happened. + + +If you are facing one of those challenges then you are already meeting the limit of :class:`~pytorch_lightning.lite.LightningLite`. +We recommend you to convert to :doc:`Lightning <../starter/new-project>`, so you never have to worry about those. + +Convert to Lightning +==================== + +The :class:`~pytorch_lightning.lite.LightningLite` is a stepping stone to transition fully to the Lightning API and benefits +from its hundreds of features. + +.. code-block:: python + + from pytorch_lightning import LightningDataModule, LightningModule, Trainer + + + class LiftModel(LightningModule): + def __init__(self, module: nn.Module): + super().__init__() + self.module = module + + def forward(self, x): + return self.module(x) + + def training_step(self, batch, batch_idx): + loss = self(batch) + self.log("train_loss", loss) + return loss + + def validation_step(self, batch, batch_idx): + loss = self(batch) + self.log("val_loss", loss) + return loss + + def configure_optimizers(self): + return torch.optim.SGD(self.parameters(), lr=0.001) + + + class BoringDataModule(LightningDataModule): + def __init__(self, dataset: Dataset): + super().__init__() + self.dataset = dataset + + def train_dataloader(self): + return DataLoader(self.dataset) + + + seed_everything(42) + model = MyModel(...) + lightning_module = LiftModel(model) + dataset = MyDataset(...) + datamodule = BoringDataModule(dataset) + trainer = Trainer(max_epochs=10) + trainer.fit(lightning_module, datamodule=datamodule) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index dda45d505b22a..4f8fbcd1fe080 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -327,7 +327,7 @@ def print(self, *args: Any, **kwargs: Any) -> None: def barrier(self) -> None: """Wait for all processes to enter this call. Use this to synchronize all parallel processes, but only if - necessary, otherwhise the overhead of synchronization will cause your program to slow down. + necessary, otherwise the overhead of synchronization will cause your program to slow down. Example:: From 2d88340a073dd0843d0bbc91ed24ab882c63de4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 23 Oct 2021 17:52:33 +0200 Subject: [PATCH 200/331] remove "mixed" --- pytorch_lightning/lite/wrappers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 34455f4ca091c..54930d6ccc116 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -109,7 +109,6 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: method.""" precision = self._accelerator.precision_plugin.precision precision_to_type = { - "mixed": torch.float16, 16: torch.float16, 32: torch.float32, 64: torch.float64, From 3171bee4c2595ba225ec441ca05bddb48d4ec5f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 23 Oct 2021 18:05:27 +0200 Subject: [PATCH 201/331] fix title levels --- docs/source/starter/lightning_lite.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/starter/lightning_lite.rst b/docs/source/starter/lightning_lite.rst index dead2f9efdd35..6fb0986dfe17a 100644 --- a/docs/source/starter/lightning_lite.rst +++ b/docs/source/starter/lightning_lite.rst @@ -41,9 +41,9 @@ Supported Integrations Coming soon: IPU accelerator, support for Horovod as a strategy and fully sharded training. -################ +**************** Learn by example -################ +**************** My existing PyTorch code ======================== From 8d3e33d9a088e8af5f2810e33d3732610b07826f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Sat, 23 Oct 2021 18:05:36 +0200 Subject: [PATCH 202/331] fix spacing --- docs/source/starter/lightning_lite.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/source/starter/lightning_lite.rst b/docs/source/starter/lightning_lite.rst index 6fb0986dfe17a..8d3bd3d1fa116 100644 --- a/docs/source/starter/lightning_lite.rst +++ b/docs/source/starter/lightning_lite.rst @@ -14,7 +14,6 @@ on any kind of device while retaining full control over their own loops and opti :class:`~pytorch_lightning.lite.LightningLite` is the right tool for you if you match one of the two following descriptions: - I want to quickly scale my existing code to multiple devices with minimal code changes. - - I would like to convert my existing code to the Lightning API, but a full path to Lightning transition might be too complex. I am looking for a stepping stone to ensure reproducibility during the transition. Supported Integrations @@ -137,11 +136,8 @@ Here are 4 required steps to convert to :class:`~pytorch_lightning.lite.Lightnin That's all. You can now train on any kind of device and scale your training. - The :class:`~pytorch_lightning.lite.LightningLite` takes care of device management, so you don't have to. - You can remove any device specific logic within your code. - Here is how to train on 8 GPUs with `torch.bfloat16 `_ precision: .. code-block:: python @@ -154,7 +150,7 @@ Here is how to use `DeepSpeed Zero3 Date: Sat, 23 Oct 2021 20:10:06 +0200 Subject: [PATCH 203/331] fix title levels --- docs/source/starter/lightning_lite.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/source/starter/lightning_lite.rst b/docs/source/starter/lightning_lite.rst index 8d3bd3d1fa116..4b2b2414421cb 100644 --- a/docs/source/starter/lightning_lite.rst +++ b/docs/source/starter/lightning_lite.rst @@ -5,8 +5,10 @@ LightningLite - Stepping Stone to Lightning .. image:: https://pl-public-data.s3.amazonaws.com/docs/static/images/lite/lightning_lite.gif :alt: Animation showing how to convert a standard training loop to a Lightning loop + :width: 600px + :align: center - +| :class:`~pytorch_lightning.lite.LightningLite` enables pure PyTorch users to scale their existing code on any kind of device while retaining full control over their own loops and optimization logic. @@ -16,8 +18,9 @@ on any kind of device while retaining full control over their own loops and opti - I want to quickly scale my existing code to multiple devices with minimal code changes. - I would like to convert my existing code to the Lightning API, but a full path to Lightning transition might be too complex. I am looking for a stepping stone to ensure reproducibility during the transition. +********************** Supported Integrations -====================== +********************** :class:`~pytorch_lightning.lite.LightningLite` supports single and multiple models and optimizers. @@ -44,6 +47,9 @@ Coming soon: IPU accelerator, support for Horovod as a strategy and fully sharde Learn by example **************** + + + My existing PyTorch code ======================== From bad83561b61b4d161432226095e57c4a0066c483 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 25 Oct 2021 10:00:06 +0200 Subject: [PATCH 204/331] add optional name to barrier Co-authored-by: four4fish <88516121+four4fish@users.noreply.github.com> --- pytorch_lightning/lite/lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 4f8fbcd1fe080..25e6cc70e1df5 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -325,7 +325,7 @@ def print(self, *args: Any, **kwargs: Any) -> None: if self.local_rank == 0: print(*args, **kwargs) - def barrier(self) -> None: + def barrier(self, name: Optional[str] = None) -> None: """Wait for all processes to enter this call. Use this to synchronize all parallel processes, but only if necessary, otherwise the overhead of synchronization will cause your program to slow down. From 8e5ddc36b50f35e13ebd8a529e3966d024b52fdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 25 Oct 2021 10:10:19 +0200 Subject: [PATCH 205/331] re-add "mixed" as it is defined in NativeMixedPrecisionPlugin --- pytorch_lightning/lite/wrappers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 54930d6ccc116..34455f4ca091c 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -109,6 +109,7 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: method.""" precision = self._accelerator.precision_plugin.precision precision_to_type = { + "mixed": torch.float16, 16: torch.float16, 32: torch.float32, 64: torch.float64, From d98713351ba050b3b884bd8be105db877204d434 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 25 Oct 2021 15:05:28 +0200 Subject: [PATCH 206/331] add lite flags section --- docs/source/starter/lightning_lite.rst | 223 ++++++++++++++++++++++--- 1 file changed, 197 insertions(+), 26 deletions(-) diff --git a/docs/source/starter/lightning_lite.rst b/docs/source/starter/lightning_lite.rst index 4b2b2414421cb..25d46979514d9 100644 --- a/docs/source/starter/lightning_lite.rst +++ b/docs/source/starter/lightning_lite.rst @@ -18,38 +18,12 @@ on any kind of device while retaining full control over their own loops and opti - I want to quickly scale my existing code to multiple devices with minimal code changes. - I would like to convert my existing code to the Lightning API, but a full path to Lightning transition might be too complex. I am looking for a stepping stone to ensure reproducibility during the transition. -********************** -Supported Integrations -********************** - -:class:`~pytorch_lightning.lite.LightningLite` supports single and multiple models and optimizers. - -.. list-table:: - :widths: 50 50 - :header-rows: 1 - - * - LightningLite arguments - - Possible choices - * - ``accelerator`` - - ``cpu``, ``gpu``, ``tpu``, ``auto`` - * - ``strategy`` - - ``dp``, ``ddp``, ``ddp_spawn``, ``ddp_sharded``, ``ddp_sharded_spawn``, ``deepspeed`` - * - ``precision`` - - ``16``, ``bf16``, ``32``, ``64`` - * - ``clusters`` - - ``TorchElastic``, ``SLURM``, ``Kubeflow``, ``LSF`` - - -Coming soon: IPU accelerator, support for Horovod as a strategy and fully sharded training. - **************** Learn by example **************** - - My existing PyTorch code ======================== @@ -284,3 +258,200 @@ from its hundreds of features. datamodule = BoringDataModule(dataset) trainer = Trainer(max_epochs=10) trainer.fit(lightning_module, datamodule=datamodule) + + + + +******************** +Lightning Lite Flags +******************** + + +Lite is a specialist for accelerated distributed training and inference. It offers you convenient ways to configure +your device and communication strategy and to seamlessly switch from one to the other. The terminology and usage is +identical to Lightning, which means minimum effort for you to convert when you decide to do so. + + +accelerator +=========== + +Choose one of ``"cpu"``, ``"gpu"``, ``"tpu"``, ``"auto"`` (IPU support is coming soon). + +.. code-block:: python + + # CPU accelerator + lite = Lite(accelerator="cpu") + + # Running with GPU Accelerator using 2 GPUs + lite = Lite(devices=2, accelerator="gpu") + + # Running with TPU Accelerator using 8 tpu cores + lite = Lite(devices=8, accelerator="tpu") + + # Running with GPU Accelerator using the DistributedDataParallel strategy + lite = Lite(devices=4, accelerator="gpu", strategy="ddp") + +The ``"auto"`` option recognizes the machine you are on, and selects the available accelerator. + +.. code-block:: python + + # If your machine has GPUs, it will use the GPU Accelerator + lite = Lite(devices=2, accelerator="auto") + + +strategy +======== + +Choose a training strategy: ``"dp"``, ``"ddp"``, ``"ddp_spawn"``, ``"tpu_spawn"``, ``"deepspeed"``, ``"ddp_sharded"``, or ``"ddp_sharded_spawn"``. + +.. code-block:: python + + # Running with the DistributedDataParallel strategy on 4 GPUs + lite = Lite(strategy="ddp", accelerator="gpu", devices=4) + + # Running with the DDP Spawn strategy using 4 cpu processes + lite = Lite(strategy="ddp_spawn", accelerator="cpu", devices=4) + + +Additionally, you can pass in your custom training type strategy by configuring additional parameters. + +.. code-block:: python + + from pytorch_lightning.plugins import DeepSpeedPlugin + + lite = Lite(strategy=DeepSpeedPlugin(stage=2), accelerator="gpu", devices=2) + + +Support for Horovod and Fully Sharded training strategies are coming soon. + + +devices +======= + +Configure the devices to run on. Can of type: + +- int: the number of GPUs to train on +- list of int: which GPUs to train on (0-indexed) +- str: a string representation of one of the above + +.. code-block:: python + + # default used by Lite, i.e., use the CPU + lite = Lite(devices=None) + + # equivalent + lite = Lite(devices=0) + + # int: run on 2 GPUs + lite = Lite(devices=2, accelerator="gpu") + + # list: run on GPUs 1, 4 (by bus ordering) + lite = Lite(devices=[1, 4], accelerator="gpu") + lite = Lite(devices="1, 4", accelerator="gpu") # equivalent + + # -1: run on all GPUs + lite = Lite(devices=-1) + lite = Lite(devices="-1") # equivalent + + + +gpus +==== + +Shorthand for setting ``devices=X`` and ``accelerator="gpu"``. + +.. code-block:: python + + # Run on 2 GPUs + lite = Lite(gpus=2) + + # Equivalent + lite = Lite(devices=2, accelerator="gpu") + + +tpu_cores +========= + +Shorthand for ``devices=X`` and ``accelerator="tpu"``. + +.. code-block:: python + + # Run on 8 TPUs + lite = Lite(gpus=8) + + # Equivalent + lite = Lite(devices=8, accelerator="tpu") + + +num_nodes +========= + + +Number of cluster nodes for distributed operation. + +.. testcode:: + + # Default used by Lite + lite = Lite(num_nodes=1) + + # Run on 8 nodes + lite = Lite(num_nodes=8) + + +Learn more about distributed multi-node training on clusters :doc:`here <../clouds/cluster>`. + + +precision +========= + +Lightning Lite supports double precision (64), full precision (32), or half precision (16) operation (including `bfloat16 `_). +Half precision, or mixed precision, is the combined use of 32 and 16 bit floating points to reduce memory footprint during model training. +This can result in improved performance, achieving significant speedups on modern GPUs. + +.. code-block:: python + + # Default used by the Lite + lite = Lite(precision=32, devices=1) + + # 16-bit (mixed) precision + lite = Lite(precision=16, devices=1) + + # 16-bit bfloat precision + lite = Lite(precision="bf16", devices=1) + + # 64-bit (double) precision + lite = Lite(precision=64, devices=1) + + +plugins +======= + +:ref:`Plugins` allow you to connect arbitrary backends, precision libraries, clusters etc. For example: +To define your own behavior, subclass the relevant class and pass it in. Here's an example linking up your own +:class:`~pytorch_lightning.plugins.environments.ClusterEnvironment`. + +.. code-block:: python + + from pytorch_lightning.plugins.environments import ClusterEnvironment + + + class MyCluster(ClusterEnvironment): + + @property + def main_address(self): + return your_main_address + + @property + def main_port(self): + return your_main_port + + def world_size(self): + return the_world_size + + + lite = Lite(plugins=[MyCluster()], ...) + + +********************** +Lightning Lite Methods +********************** \ No newline at end of file From 25c5b9929fddb0ce8461de45d792df982a00d267 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Oct 2021 13:06:50 +0000 Subject: [PATCH 207/331] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/source/starter/lightning_lite.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/source/starter/lightning_lite.rst b/docs/source/starter/lightning_lite.rst index 25d46979514d9..db7622787b2a0 100644 --- a/docs/source/starter/lightning_lite.rst +++ b/docs/source/starter/lightning_lite.rst @@ -347,11 +347,11 @@ Configure the devices to run on. Can of type: # list: run on GPUs 1, 4 (by bus ordering) lite = Lite(devices=[1, 4], accelerator="gpu") - lite = Lite(devices="1, 4", accelerator="gpu") # equivalent + lite = Lite(devices="1, 4", accelerator="gpu") # equivalent # -1: run on all GPUs lite = Lite(devices=-1) - lite = Lite(devices="-1") # equivalent + lite = Lite(devices="-1") # equivalent @@ -436,7 +436,6 @@ To define your own behavior, subclass the relevant class and pass it in. Here's class MyCluster(ClusterEnvironment): - @property def main_address(self): return your_main_address @@ -454,4 +453,4 @@ To define your own behavior, subclass the relevant class and pass it in. Here's ********************** Lightning Lite Methods -********************** \ No newline at end of file +********************** From 86c163e4fbf83323971a8d3f667807ddf05b4593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 25 Oct 2021 15:08:06 +0200 Subject: [PATCH 208/331] Update docs/source/starter/lightning_lite.rst Co-authored-by: Nicki Skafte Detlefsen --- docs/source/starter/lightning_lite.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/starter/lightning_lite.rst b/docs/source/starter/lightning_lite.rst index db7622787b2a0..e01a1a5fc83dc 100644 --- a/docs/source/starter/lightning_lite.rst +++ b/docs/source/starter/lightning_lite.rst @@ -117,7 +117,7 @@ Here are 4 required steps to convert to :class:`~pytorch_lightning.lite.Lightnin That's all. You can now train on any kind of device and scale your training. The :class:`~pytorch_lightning.lite.LightningLite` takes care of device management, so you don't have to. -You can remove any device specific logic within your code. +You should remove any device specific logic within your code. Here is how to train on 8 GPUs with `torch.bfloat16 `_ precision: .. code-block:: python From ae1d79392a700b088b5213a1b51c4679f3dd401d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 25 Oct 2021 16:42:22 +0200 Subject: [PATCH 209/331] Add documentation for essential methods --- docs/source/starter/lightning_lite.rst | 107 +++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/docs/source/starter/lightning_lite.rst b/docs/source/starter/lightning_lite.rst index e01a1a5fc83dc..7f03b7c34ed97 100644 --- a/docs/source/starter/lightning_lite.rst +++ b/docs/source/starter/lightning_lite.rst @@ -454,3 +454,110 @@ To define your own behavior, subclass the relevant class and pass it in. Here's ********************** Lightning Lite Methods ********************** + + +run +=== + +The run method servers two purposes: + +1. Override this method from the :class:`~pytorch_lightning.lite.lite.LightningLite` class and put your + training (or inference) code inside. +2. Launch the training by calling the run method. Lite will take care of setting up the distributed backend. + +You can optionally pass arguments to the run method. For example, the hyperparameters or a backbone for the model. + +.. code-block:: python + + from pytorch_lightning.lite import LightningLite + + class Lite(LightningLite): + + # Input arguments are optional, put whatever you need + def run(self, learning_rate, num_layers): + # Here goes your training loop + + lite = Lite(accelerator="gpu", devices=2) + lite.run(learning_rate=0.01, num_layers=12) + + +setup +===== + +Setup a model and corresponding optimizer(s). If you need to setup multiple models, call ``setup()`` on each of them. +Moves the model and optimizer to the correct device automatically. + +.. code-block:: python + + model = nn.Linear(32, 64) + optimizer = torch.optim.SGD(model.parameters(), lr=0.001) + + # Setup model and optimizer for accelerated training + model, optimizer = self.setup(model, optimizer) + + # If you don't want Lite to set the device + model, optimizer = self.setup(model, optimizer, move_to_device=False) + + +The setup method also prepares the model for the selected precision choice so that operations during ``forward()`` get +cast automatically. + +setup_dataloaders +================= + +Setup one or multiple dataloaders for accelerated operation. If you are running a distributed plugin (e.g., DDP), Lite +will replace the sampler automatically for you. In addition, the dataloader will be configured to move the returned +data tensors to the correct device automatically. + +.. code-block:: python + + train_data = torch.utils.DataLoader(train_dataset, ...) + test_data = torch.utils.DataLoader(test_dataset, ...) + + train_data, test_data = self.setup_dataloaders(train_data, test_data) + + # If you don't want Lite to move the data to the device + train_data, test_data = self.setup_dataloaders(train_data, test_data, move_to_device=False) + + # If you don't want Lite to replace the sampler in the context of distributed training + train_data, test_data = self.setup_dataloaders(train_data, test_data, replace_sampler=False) + + +backward +======== + +This replaces any occurences of ``loss.backward()`` and will make your code accelerator and precision agnostic. + +.. code-block:: python + + output = model(input) + loss = loss_fn(output, target) + + # loss.backward() + self.backward(loss) + + +to_device +========= + +Use :class:`~pytorch_lightning.lite.lite.LightningLite.to_device` to move models, tensors or collections of tensors to +the current device. By default :class:`~pytorch_lightning.lite.lite.LightningLite.setup` and +:class:`~pytorch_lightning.lite.lite.LightningLite.setup_dataloaders` already move the model and data to the correct +device, so calling this method is only necessary for manual operation when needed. + +.. code-block:: python + + data = torch.load("dataset.pt") + data = self.to_device(data) + + +print +===== + +Print to the console via the built-in print function, but only on the main process. + + +.. code-block:: python + + # Print only on the main process + self.print(f"{epoch}/{num_epochs}| Train Epoch Loss: {loss}") From 9382fa4e0875afc1dafa59299deb5d48004f0678 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 25 Oct 2021 16:43:05 +0200 Subject: [PATCH 210/331] add spacers --- docs/source/starter/lightning_lite.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/source/starter/lightning_lite.rst b/docs/source/starter/lightning_lite.rst index 7f03b7c34ed97..6d733f2001ab1 100644 --- a/docs/source/starter/lightning_lite.rst +++ b/docs/source/starter/lightning_lite.rst @@ -19,6 +19,9 @@ on any kind of device while retaining full control over their own loops and opti - I would like to convert my existing code to the Lightning API, but a full path to Lightning transition might be too complex. I am looking for a stepping stone to ensure reproducibility during the transition. +---------- + + **************** Learn by example **************** @@ -65,6 +68,10 @@ The ``run`` function contains custom training loop used to train ``MyModel`` on run(10) + +---------- + + Convert to LightningLite ======================== @@ -181,6 +188,9 @@ Here is an example while running on 256 GPUs. .. note:: We recommend instantiating the models within the :meth:`~pytorch_lightning.lite.LightningLite.run` method as large models would cause OOM Error otherwise. +---------- + + Distributed Training Pitfalls ============================= @@ -209,6 +219,8 @@ but there are several major challenges ahead of you now: If you are facing one of those challenges then you are already meeting the limit of :class:`~pytorch_lightning.lite.LightningLite`. We recommend you to convert to :doc:`Lightning <../starter/new-project>`, so you never have to worry about those. +---------- + Convert to Lightning ==================== @@ -260,6 +272,7 @@ from its hundreds of features. trainer.fit(lightning_module, datamodule=datamodule) +---------- ******************** @@ -451,6 +464,9 @@ To define your own behavior, subclass the relevant class and pass it in. Here's lite = Lite(plugins=[MyCluster()], ...) +---------- + + ********************** Lightning Lite Methods ********************** From 219de459ab7d02d2977d90ee51dbb14f65fe6574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 25 Oct 2021 16:56:14 +0200 Subject: [PATCH 211/331] add save_checkpoint, barrier to docs --- docs/source/starter/lightning_lite.rst | 32 ++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/source/starter/lightning_lite.rst b/docs/source/starter/lightning_lite.rst index 6d733f2001ab1..ffbc2e7963e06 100644 --- a/docs/source/starter/lightning_lite.rst +++ b/docs/source/starter/lightning_lite.rst @@ -577,3 +577,35 @@ Print to the console via the built-in print function, but only on the main proce # Print only on the main process self.print(f"{epoch}/{num_epochs}| Train Epoch Loss: {loss}") + + +save_checkpoint +=============== + +Save contents to a checkpoint. Replaces all occurences of ``torch.save(...)`` in your code. Lite will take care of +handling the saving part correctly, no matter if you are running single device, multi-device or multi-node. + +.. code-block:: python + + # Instead of `torch.save(...)`, call: + self.save_checkpoint("path/to/checkpoint.ckpt", model.state_dict()) + + +barrier +======= + +Call this if you want all processes to wait and synchronize. Once all processes have entered this call, +execution continues. Useful for example when you want to download data on one process and make all others wait until +the data is written to disk. + +.. code-block:: python + + # Download data only on one process + if self.global_rank == 0: + download_data("http://...") + + # Wait until all processes meet up here + self.barrier() + + # All processes are allowed to read the data now + From 66a3f1fbab3b428ed436972bc05284ab8c41f833 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Oct 2021 16:26:17 +0000 Subject: [PATCH 212/331] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/source/starter/lightning_lite.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/starter/lightning_lite.rst b/docs/source/starter/lightning_lite.rst index ffbc2e7963e06..e4fc940a28a62 100644 --- a/docs/source/starter/lightning_lite.rst +++ b/docs/source/starter/lightning_lite.rst @@ -608,4 +608,3 @@ the data is written to disk. self.barrier() # All processes are allowed to read the data now - From 16ac4df38fc546b0b07cabbc714eb353c7c1b04d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 25 Oct 2021 18:36:02 +0200 Subject: [PATCH 213/331] update lite with latest master changes --- pytorch_lightning/lite/lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 25e6cc70e1df5..f090127b60818 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -183,7 +183,7 @@ def setup( param_group["params"] = [mapping.get(p, p) for p in param_group["params"]] # Let accelerator/plugin wrap and connect the models and optimizers - [model], optimizers = self._strategy._setup_models_and_optimizers([model], list(optimizers)) + model, optimizers = self._strategy._setup_model_and_optimizers(model, list(optimizers)) model = _LiteModule(model, self._accelerator) optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self._accelerator) for optimizer in optimizers] self._num_models += 1 From debe472880f0c859ad373601d1b46e4c1d67a7a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 25 Oct 2021 18:44:39 +0200 Subject: [PATCH 214/331] remove unused method in tpu spawn --- pytorch_lightning/plugins/training_type/tpu_spawn.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pytorch_lightning/plugins/training_type/tpu_spawn.py b/pytorch_lightning/plugins/training_type/tpu_spawn.py index 7df2ff16e61f6..2400bdb3ffc34 100644 --- a/pytorch_lightning/plugins/training_type/tpu_spawn.py +++ b/pytorch_lightning/plugins/training_type/tpu_spawn.py @@ -184,9 +184,6 @@ def new_process(self, trainer: "pl.Trainer", mp_queue: SimpleQueue) -> None: def model_to_device(self) -> None: self.model = self.wrapped_model.to(self.root_device) - def setup_model(self, model: Module) -> Module: - return model - def barrier(self, name: Optional[str] = None) -> None: if self.is_distributed: rendezvous(name) From f3cb16317b72ec4ebdd51621915adf52becf9c43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 25 Oct 2021 18:44:57 +0200 Subject: [PATCH 215/331] remove unused import --- tests/lite/test_lite.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index eb21b71177121..def9ce29ac9dc 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -21,7 +21,6 @@ import torch.distributed import torch.nn.functional from torch import nn -from torch.optim import Optimizer from torch.utils.data import DataLoader, DistributedSampler, Sampler from pytorch_lightning import seed_everything From 168738754e5ffdbf76265fc897a7aa2a61b976d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 25 Oct 2021 18:46:24 +0200 Subject: [PATCH 216/331] fix precommit formatting issue --- docs/source/starter/lightning_lite.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/starter/lightning_lite.rst b/docs/source/starter/lightning_lite.rst index e4fc940a28a62..b3ea663ece039 100644 --- a/docs/source/starter/lightning_lite.rst +++ b/docs/source/starter/lightning_lite.rst @@ -491,7 +491,8 @@ You can optionally pass arguments to the run method. For example, the hyperparam # Input arguments are optional, put whatever you need def run(self, learning_rate, num_layers): - # Here goes your training loop + """Here goes your training loop""" + lite = Lite(accelerator="gpu", devices=2) lite.run(learning_rate=0.01, num_layers=12) From 5d0a72b4446f0953d13bb9f997a6c49c46d5d0e3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Oct 2021 16:47:42 +0000 Subject: [PATCH 217/331] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/source/starter/lightning_lite.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/starter/lightning_lite.rst b/docs/source/starter/lightning_lite.rst index b3ea663ece039..20b9cc75dce2d 100644 --- a/docs/source/starter/lightning_lite.rst +++ b/docs/source/starter/lightning_lite.rst @@ -487,6 +487,7 @@ You can optionally pass arguments to the run method. For example, the hyperparam from pytorch_lightning.lite import LightningLite + class Lite(LightningLite): # Input arguments are optional, put whatever you need From 7906eb304374dc9b0b18ffa6bf976dc5c85cae97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 25 Oct 2021 22:34:01 +0200 Subject: [PATCH 218/331] remove reduce_decision and execute_on_rank --- pytorch_lightning/lite/lite.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index f090127b60818..88b9efc455d82 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -342,21 +342,6 @@ def barrier(self, name: Optional[str] = None) -> None: """ self._strategy.barrier() - def reduce_decision(self, decision: bool) -> bool: - """Reduce a boolean decision across processes. - - Use this for example to determine an early stopping condition, in which case you want to stop if any of - the processes determine to stop. - - Args: - decision: The decision on the current process - - Return: - If at least one of the processes enters with ``decision=True``, then all processes will return `True`. - Otherwise returns ``False``. - """ - return self._strategy.reduce_boolean_decision(decision) - def all_gather( self, data: Union[torch.Tensor, Dict, List, Tuple], group: Optional[Any] = None, sync_grads: bool = False ) -> Union[torch.Tensor, Dict, List, Tuple]: @@ -391,19 +376,6 @@ def save_checkpoint(self, filepath: Union[str, Path], content: Dict[str, Any]) - """ self._strategy.save_checkpoint(content, filepath) - def execute_on_rank(self, func: Callable, rank: int, *args: Any, **kwargs: Any) -> None: - """Execute the given function only on the given process. - - Args: - func: The function to execute - rank: The index of the process across all devices and nodes (global rank). This value must be an integer - in the range ``[0, self.world_size - 1]``. - *args: Optional positional arguments passed to the function - **kwargs: Optional named arguments passed to the function - """ - if self.global_rank == rank: - func(*args, **kwargs) - def _run_wrapper(self, run_method: Callable) -> Callable: return partial(self._run_impl, run_method) From 0482860c951b0db814a7f4d72deaa8464681544f Mon Sep 17 00:00:00 2001 From: tchaton Date: Tue, 26 Oct 2021 10:34:04 +0100 Subject: [PATCH 219/331] update on comments --- docs/source/starter/lightning_lite.rst | 2 ++ pytorch_lightning/lite/lite.py | 33 ++++++++++++++++---------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/docs/source/starter/lightning_lite.rst b/docs/source/starter/lightning_lite.rst index 20b9cc75dce2d..ed60ec63c284e 100644 --- a/docs/source/starter/lightning_lite.rst +++ b/docs/source/starter/lightning_lite.rst @@ -19,6 +19,8 @@ on any kind of device while retaining full control over their own loops and opti - I would like to convert my existing code to the Lightning API, but a full path to Lightning transition might be too complex. I am looking for a stepping stone to ensure reproducibility during the transition. +.. note:: LightningLite is a Lightning beta feature and will be considered stable for v1.6. + ---------- diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 88b9efc455d82..f94809afe626f 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -13,11 +13,10 @@ # limitations under the License. import os from abc import ABC, abstractmethod -from collections import Callable from contextlib import contextmanager from functools import partial from pathlib import Path -from typing import Any, Dict, Generator, Iterable, List, Optional, overload, Sequence, Tuple, Union +from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, overload, Sequence, Tuple, Union import torch import torch.nn as nn @@ -170,17 +169,7 @@ def setup( self._validate_setup(model, optimizers) if move_to_device: - params_on_cpu = dict(model.named_parameters()) - model = self.to_device(model) - params_on_device = dict(model.named_parameters()) - - # When the user creates the optimizer, they reference the parameters on the CPU. - # However, when running with TPU the parameters get copied and the reference in the optimizer - # remains invalid. We need to update the references to point to the parameter tensors on the device. - mapping = {param: params_on_device[name] for name, param in params_on_cpu.items()} - for optimizer in optimizers: - for param_group in optimizer.param_groups: - param_group["params"] = [mapping.get(p, p) for p in param_group["params"]] + model = self._move_model_to_device(model=model, optimizers=list(optimizers)) # Let accelerator/plugin wrap and connect the models and optimizers model, optimizers = self._strategy._setup_model_and_optimizers(model, list(optimizers)) @@ -402,7 +391,25 @@ def _set_plugin_specific_precision_variables(self) -> None: if isinstance(self._strategy, DDPShardedPlugin): self._strategy._precision = self._accelerator_connector.precision + def _move_model_to_device(self, model: nn.Module, optimizers: List[Optimizer]) -> nn.Module: + if isinstance(self._strategy, TPUSpawnPlugin): + # When the user creates the optimizer, they reference the parameters on the CPU. + # However, when running with TPU the parameters get copied and the reference in the optimizer + # remains invalid. We need to update the references to point to the parameter tensors on the device. + params_on_cpu = dict(model.named_parameters()) + model = self.to_device(model) + params_on_device = dict(model.named_parameters()) + + mapping = {param: params_on_device[name] for name, param in params_on_cpu.items()} + for optimizer in optimizers: + for param_group in optimizer.param_groups: + param_group["params"] = [mapping.get(p, p) for p in param_group["params"]] + else: + model = self.to_device(model) + return model + def _set_deepspeed_precision_variables(self) -> None: + # TODO: Refactor this once precision pluging is part of the strategy. amp_type = self._accelerator_connector.amp_type amp_level = self._accelerator_connector.amp_level precision = self._accelerator_connector.precision From 2359d0c230b747c85d412a2b0b085c073df31369 Mon Sep 17 00:00:00 2001 From: Kaushik B Date: Tue, 26 Oct 2021 17:15:12 +0530 Subject: [PATCH 220/331] Add barrier for TPU Spawn --- pytorch_lightning/plugins/training_type/tpu_spawn.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pytorch_lightning/plugins/training_type/tpu_spawn.py b/pytorch_lightning/plugins/training_type/tpu_spawn.py index 2400bdb3ffc34..eeaf46caf448e 100644 --- a/pytorch_lightning/plugins/training_type/tpu_spawn.py +++ b/pytorch_lightning/plugins/training_type/tpu_spawn.py @@ -31,6 +31,7 @@ from pytorch_lightning.trainer.connectors.data_connector import DataConnector from pytorch_lightning.trainer.states import TrainerFn from pytorch_lightning.utilities import _TPU_AVAILABLE, find_shared_parameters, rank_zero_warn, set_shared_parameters +from pytorch_lightning.utilities.apply_func import move_data_to_device from pytorch_lightning.utilities.data import has_len from pytorch_lightning.utilities.distributed import rank_zero_only, ReduceOp from pytorch_lightning.utilities.exceptions import MisconfigurationException @@ -266,6 +267,15 @@ def spawn(self, function: Callable, *args: Any, **kwargs: Any) -> Any: xmp.spawn(self._wrapped_function, args=(function, args, kwargs, return_queue), **self.get_mp_spawn_kwargs()) return return_queue.get() + def _wrapped_function( + self, process_idx: int, function: Callable, args: Any, kwargs: Any, return_queue: SimpleQueue + ) -> None: + self._worker_setup(process_idx) + result = function(*args, **kwargs) + if self.local_rank == 0: + return_queue.put(move_data_to_device(result, "cpu")) + self.barrier("end-process") + def _worker_setup(self, process_idx: int): reset_seed() self.tpu_local_core_rank = xm.get_local_ordinal() From d56608216761cffbe6e22ca0ffd3691ea488b603 Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Tue, 26 Oct 2021 18:23:19 +0100 Subject: [PATCH 221/331] Add Mnist examples with lite (#10131) Add MNIST PyTorch to Lightning examples --- .gitignore | 2 + grid_generated_0.png | Bin 0 -> 7000 bytes grid_ori_0.png | Bin 0 -> 1219 bytes pl_examples/README.md | 22 +- pl_examples/basic_examples/README.md | 57 ++-- .../basic_examples/mnist_examples/README.md | 67 +++++ .../mnist_examples}/__init__.py | 0 .../image_classifier_1_pytorch.py | 160 +++++++++++ .../mnist_examples/image_classifier_2_lite.py | 131 +++++++++ .../image_classifier_3_lite_to_lightning.py | 166 ++++++++++++ .../image_classifier_4_lightning.py | 86 ++++++ ...image_classifier_5_lightning_datamodule.py | 92 +++++++ .../basic_examples/simple_image_classifier.py | 82 ------ .../generative_adversarial_net.py | 38 +-- pl_examples/integration_examples/__init__.py | 0 .../dali_image_classifier.py | 0 .../pytorch_2_lite_2_lightning.py | 248 ------------------ pl_examples/loop_examples/mnist_lite.py | 189 +++++++++++++ pl_examples/run_examples.sh | 19 +- pl_examples/test_examples.py | 2 +- 20 files changed, 973 insertions(+), 388 deletions(-) create mode 100644 grid_generated_0.png create mode 100644 grid_ori_0.png create mode 100644 pl_examples/basic_examples/mnist_examples/README.md rename pl_examples/{lite_examples => basic_examples/mnist_examples}/__init__.py (100%) create mode 100644 pl_examples/basic_examples/mnist_examples/image_classifier_1_pytorch.py create mode 100644 pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py create mode 100644 pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py create mode 100644 pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py create mode 100644 pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py delete mode 100644 pl_examples/basic_examples/simple_image_classifier.py create mode 100644 pl_examples/integration_examples/__init__.py rename pl_examples/{basic_examples => integration_examples}/dali_image_classifier.py (100%) delete mode 100644 pl_examples/lite_examples/pytorch_2_lite_2_lightning.py create mode 100644 pl_examples/loop_examples/mnist_lite.py diff --git a/.gitignore b/.gitignore index 7b1247433e7b4..4229c050e9b7f 100644 --- a/.gitignore +++ b/.gitignore @@ -137,6 +137,8 @@ ENV/ Datasets/ mnist/ legacy/checkpoints/ +*.gz +*ubyte # pl tests ml-runs/ diff --git a/grid_generated_0.png b/grid_generated_0.png new file mode 100644 index 0000000000000000000000000000000000000000..77820f68637fd9ace9e72d03a3af6e1cd155db03 GIT binary patch literal 7000 zcmW+*2{@G9+ne9+ubW){ho`Z~u~k7qAuuTDfm%|F zgOLnH73n(pJt-z8hL8Quz^4c2&!1-vJAgn4a;z;ih-wC`xxhvch}mA+83_sd<`-w- z!%>luhq<}koSZ^yM^usXbA2TaMbw7T@yW?UtgQOiuU|ku4G%vUQG3JO+}zYO0xPa^ zuh(mR-YQAoQ8}?DKi|5tx7C` z60)-t!I)y93v#f==$PKTSyx~0@8@S#6 z&9i6E?rsn5ySln^9XS%PJaOH@VPd*7`^=d$q8I&F*5)jbT$Apq-7d-$J(?L>UU0TS zG^E0Pgo%ae92IX@*xlSb1ABpMUAWWq=3;=x#ft}-YTDWe`ubc*E>jr_0>Q$<@)#@5 z9R2##I`tG7UPlNrM`Hv_N(qlu8F#W z!tAOVYbWtlt zJzy=Tq{PSHf9t8R`g>!dx3_Y*_R`v#G8M0v?s@m_;fPvENy)PE@+hqMv)3~Kw(U;b40j`LJ!Af;Bnu9BX8Nnke zD#yd7ckP;-Whq<^JSuL3crl(IKYrjOo+l(EK;sZ;-sMR+2`MS5)OIJ57UKE-V7z1N zn+INAu$%%lx^@K34?Axs(%ReG{VT0ee8JmuC7GF-!*_1pyeS}O%@>KickdpN_9-nb zta%(0vR6@5q=>;(Wo2bmS68Q|GD-M&dq0hi?w<25kWV4JHu_7kIE*T;bF6tH@kA>%Ju6pSaB#(rcT=3yLUH-!?*#x zY^nlJAd!|PCO`Z70{)EA*PABC=y5K}0S*p2`CZw@Sp(%hCB?;NxhAHjrfKbuwZt0j z6-Zj$6@K^O76^oA+1WEsj-SDb9~Fwp%g#P>_%NAnFBNwBs^t09W7(%Qz{oE`C* za>i7UXL1rd>U_h8P4CeC$2PH+(ELE>aV9v*pBRoFoF)^s+U=4V%zqF{8C zfSiu5Zb?arE*1+lqT=7RJ3%vj-gxrliPneO5vY?Q0&(={QJe%6H%?v`>6j*5viIHRuHELc4yObg?^Erzc5l8(c(~~mritY`fMwYmx^nh1OO3Hb+>JKbso^u+S(dAl7pLj zf4Noj7%y+MP8wdSsBdm+Y6^jH8LGOdsCZ1G_H%4XO3IfnU+8ptS=j{}o1w);cNAZ5 zPY*RAVSTnoF-d=Na?%F~Wn;rPgW)&bA)RTwwY56)?wu3@F+DvkE-tREt^MfHBbdq> zTUGB_L6{qQZEbDe$9r*fdqAk*;9yunIWadsKelr;Y9K1N$%iCKAi&ahfo~YuswY_}^fV$33kV6^nbZ5f@ z$+iCG-7}m76A4};e(Drd)C>(kxVOSE0DOQREpQ!Bi^M+EPJz>jXa)mN| zgC!3poK0ZE7B(1b|Erlfgg_)WA_wO^`G?I$DbLWoa>C>=P{wT?W zX3!|b1qFm#w>IkpY+!z9&RtKisf&w?BO@aqNls~K1a7a*`B#dNwBTObzEmVgD+!&I zka$~A@T_*ExTFM55`|3#7#N?J2wv-P4JPuU<9Ud*X17 z&`zNLfW4`B*6|DH&fO3vSM<3G#eBTe`~p^lX%*n-*EcY*Dfhm|cy~Qf`JVsMuPaVY z?{N}pL0cqe|eW%6cbOkHxXhJ)9?rQ3f3Jz!_|Yj=D7 zs>E}=)bjeeX3&G#9}8_Zh*=J%F{^$l)f!5=;r3e8@WP1 zo$TxDi$;#JE;RaKQw&+hs=KxB#@g-X4E!Ei;?`e|5$ZV-xj$C&yr z3)(4J)#HncvPJGsK(08tv5AQ{s2;0wZ;-%o^s)Ya%W!RvkuTx*`aWKj;04-xPSH!p zOP!IG6*+nGzAAEeclX4J6Cj#((tvez(+!t=D=tTHN%OH6)O{XV1f&NAm2XoWT>FR< zD}L_$`I{LGs|sI_&a9iKPMrdv5cBi*H_u?)v9}K$H3J5{bcw{60{MhGdzNONJ1{T+ z%2*+>ML$CkemB1>WPQNr!Gj0oJ`2_*uEG>Ot0IROyi~d0^2Cd?7A#DcEG_#3UNMDh z!weW(SX@gX5KzYY`rqN+0CCOTJBxBOs zodPjK495M=mpQQl_`dtb%hvj zqyqI@S64#qh;3o_LiK(ykXcYrkl3c%@*1>NR~LwAe+6oVUEP!Wlixx81Dp6~fF0=V z?ai;+%KGr(2dEhr7qq-QPrteXRYOCABdi9_)8@1SP1o+k(#(n9{`Bb+3~;l(f_qm} zb8}6^i?}$(z(C80JCR7_62$~=ERL+KthBe^kfG>hFw#;}!2^NyILm(xXp4ztW@d&s z?&0md_A}Q4fza?>{I9BN5b)m5uh1+fIe04)*xyB&KkEF23nz{pV{~*ByA4&voi=!g ziP6u1neWUtJr?B0?u7CeqU-QwSTs+fTK*!c>11y4u>boE#T;nPH|^aIm#( z!v0#H+kR*K(vk;YotN7BTpu9B2fI4+3!q9+5FIU0a;QAz@x30?QhM^QO)C7##x|? z`k^){DYwBmuP+Q~2JiUl>K^iC3dM?pD%VZ{(+u702i1Tf2TI-9-G%0bThi8M!UgYc z(SYjB&^Ccf2a#MLx=gY=!IiI5hX%F?YXtd+e z7cO2*(oPv(^5u`B`L9l|H#z$HmVWwVp4|y8ssob)rdvVbheOd_nmHV3(GFy&ib})e zatgu8+uPgCO|JV$$-TaJhM6o(B$MoL?UbdZCGC{9DKC+W{(K08yU3wKzaKw-4D-dy z#=;(+YAkb@Mccrj6=s=*=>QAU)cm|oT04lYxw$#Klo6VLp}X+1IQh4iS{u>P!2vY3 zscDy6)h6qTxr%E)H2UJ&8W@(G++0p>?vlPawWOzd>0F$gaZk8Y27{Rkqn#p; zM(4mCz_hZ4T|)5@m4pNY5`avgI^SDc!ypv|=;Yz*Wlkk7~n zbOETt7#+aCsF)bQ&wF?89*x8TlDOZ$59J3_0!so$glqr9!&cDsvc*9;v9h$(7e)pu zcQIhCqR0&T#9g(&t4n~S_3hiYjg1YE7BK!WeK3Y7ik<@1g=W6Hv-2363NuE=K_7Uj zC0)IGl}Pg^60-^l7MDgFK&b%kJO9k7u`677ko@c!4f(K$7AXD-U2pFmgO2kWLcM@TbypbR_EiEkiKv4GgBZY+6p9A`QK6(NV zsY)r#B?#rc<>dX@xw(VapMjjwPC>ZU*eje9#Dm8($p!$?Nm~LnXz)g(EO7G==s>7? zWktnbU1eiw3HswGAAz&5cmWHY)aj4yShHD zvpcV(RN=dn3Z_C*@(wh&tLr?h0la8sWd)$tL8O5K0Yrz%)fFehBP$ys;1CqF3jqtj zH%bL)3j`Q^6gFuhLjlqU$ygP<>kmv~Xvmw~3Mw1)4qjiMpN&N{WT%jT8uKjEjTNbU zFvA~-wUZZwbU^~rv7Dl!`;eLeqM2l`&h`}XM%shs_;2!QOiZQMlp(lj$Q}X%cUD%4 z_0qF)b3wZU<5+xjA>$+t95_IsP<-b5J?`I+ij4(B>u+ELh5-;tH?2J@JNr^%a(uj! zqGFNfcoXnNxc2R$p47Co0Du1+6K7z%7@ai3Ot3>5BHWQ!Su`5@>h*>D6JYi275o4J z5s1(yf`WpO8tSVh4fghKEsoSiM@NHZ3D+ja$J;9j!Bq0|^QWe!zIyelz|6(pUyY>2 zcI3!OVCkKWLdI0F8JfXh96x^CCU_Hk^3Ja)IY`VjH-?!J2vC-gAI}0xf#_JSaDkL9EtQaztOR-XnZ#V;Iv~y)32z{g zNRX)62A_zj2VLLP1keFPI1ahsqww&`8D}_PL#!zDJ7h-YxyqWFq9iRac>fXJ!|7xb zVCqA_SsG>NLp%#}ps871Q*&uil^7iShHk$HCx7wz%Z4kyNibn3;swX*5K+@ zYEn{5Vbga zzg;2Lq1*fK?fmtt+gn@jji>*NH$&{n$$A;~g+M@4V#Nz{b8~ZYW>;srAOaHO3Ei?Q z>HGiK$5@79WMt%D84U7KSXdZBZck4LgFrukkB^U!hnxnS{)oGEr9X(*@2@W~*F#_m z$tIj)rm6}PNADOKS{NS>ud)4)fQrO=`uH5W_;q%6c5inZ=Fder5!l=?Q!4JRySu!i z;(af#c&zyN*jT9JUBH(87C@^6GWjSI$$fdE1*Au=Ng|cn^YiCV2IGGveM$OYPvUVB zAZ|c{G>ldTZ38bwgooci@ey81@*dt8L;%%^@x8 zsWyy)EQ4@2N#D)c85oK|q=Cwds)Mc$*%=iD4TF;amd@_f6DJRn1q1{j&;ywf6jVt- zDG9|uQU}Zg*a6`f0ma8e`UZN2h2?8Q!?JG$j0Fo*p-Yc~JgvF884LxiBNFSQiiAWK zcyEDXprwWIy$17KXtwib{2G!g0a{p4knP3%=JF)e-NM`)Qk!UOJ3E8R(vUWZlN*UN$Qf)4&En`@6W^$iaU5g&7x+n9MFnhS0)^sH)(_GantsVw zqs;PY?FeMm?>dPvxDX1Ddv0!Qs3z${D3U<8hlA&L$(=n5%g1P^bh~`i1StZR3ib`O zFzjn`^Cl%BVRB(XR7j||tLt34?)uQ^(gtjDa(w)@{7bNN`DTR=LDlH$R{O8Avjp$1b`fd4t*zXz uIqK`{L8ry&Fb^HraocsAVGoz2@>Rw zVrFK>>-APwR|i_y*4E~5I9@+ua&l6sRDOrs-vNiiF&d2kfb;Y7fC`(Mn!uOXY___( zy6t9Qd{0>{R(W~(;Nal% z^Yg^S1a5mo_zPcIYHI5C_BNGDeSCZ@Dk@rATf<&Ur_*0vUS3~cKYSV;9lg4`DiVpN zrlu+?DzFIh`Fx2)l97?|S%Z*}5V>5AAV_z2cW-YmcB_GrnVAWPy4h?dQI^eSZ)|J; z0PJ>qPEHQSs*#Zq&l>;$k00-T9v&WI5yUfnI2=xYe?Po78jUyhl9uC z<>lqY#l}`UcR}xiBYGqv9YtW6J94JB?$xq9*<|WTIqCpWMt&o+1bj< zidZbx>2$cQ{yQ@>GjN#b_4=@|FdPbng@p!#;p*zjY&IVq9a$_E|An@`zMgc?5{bm) z@jN^{2!%q@#Su1}y}!Q?-4_=Z;}f~NyZd7I+1c62&(9~m0m4%`x z002eN($Z4gI$SPS{6=D8qVK*?D3r-$zQ}fWcgb*#Aa5fD<3bw%QBhG#OH0u1?CdPb z>VAZVhDxQ<>+5Tm%eB0`92*;pLxe`7<>lq|^z?Lhcel53)J(EVkKf005Org|TE{U|@K7xS*gwtJSWruYa~fqbMp62uQFP z5fO26a`Jt2w~C zXMTQOC=~wq@q+|&;o;$ni;FOkwY4?>Q+c4&*4DPPv~amxERC5==KcLW0N~)@0C$fN zxX?~dPZQMnh3^&a>FFswJsr1>TrPLJ-2i~&`1N+Ae>EG#Ve&JHOl zDFoG;nwoBJZ{eEf3wmg1C@n1wrV|zx#$vHpELK870!#z|(AwHcuwNJo03bIvm!K3v zqtUEZD|Dt%D2U3vp-?Es#>P+-g|o@G&8t+Zv$L~z2i&QtDdIgc7>v`?Q*uISXlQ`W z_V)JJY&LOo@2IJ%866#UI-UMwhx9}sCX>nQ^-85uqC)BE>F@|k;xPG;s;VlDM&lpH z+}s@TS>q3!jAU*={{svL1IAGP{ta&aBlPw4Aqc`|vuQNi*DdjXqy6L1>-E#q)0dZ* hpM41p5+n$5egYD7Ldw>)hDQJZ002ovPDHLkV1kjyQ?md7 literal 0 HcmV?d00001 diff --git a/pl_examples/README.md b/pl_examples/README.md index afc2b06873ea2..978434787ee1c 100644 --- a/pl_examples/README.md +++ b/pl_examples/README.md @@ -5,17 +5,31 @@ can be found in our sister library [lightning-bolts](https://pytorch-lightning.r ______________________________________________________________________ -## Basic examples +## MNIST Examples -In this folder we add 3 simple examples: +5 MNIST examples showing how to gradually convert from pure PyTorch to PyTorch Lightning. + +The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst) from pure PyTorch is optional but it might helpful to learn about it. + +- [MNIST with vanilla PyTorch](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/basic_examples/mnist_examples/image_classifier_1_pytorch.py) +- [MNIST with LightningLite](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py) +- [MNIST LightningLite to LightningModule](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py) +- [MNIST with LightningModule](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py) +- [MNIST with LightningModule + LightningDataModule](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py) + +______________________________________________________________________ + +## Basic Examples + +In this folder, we add 2 simple examples: -- [MNIST Classifier](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/basic_examples/simple_image_classifier.py) (defines the model inside the `LightningModule`). - [Image Classifier](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/basic_examples/backbone_image_classifier.py) (trains arbitrary datasets with arbitrary backbones). +- [Image Classifier + DALI](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/basic_examples/mnist_examples/image_classifier_4_dali.py) (defines the model inside the `LightningModule`). - [Autoencoder](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/basic_examples/autoencoder.py) (shows how the `LightningModule` can be used as a system) ______________________________________________________________________ -## Domain examples +## Domain Examples This folder contains older examples. You should instead use the examples in [lightning-bolts](https://pytorch-lightning.readthedocs.io/en/latest/ecosystem/bolts.html) diff --git a/pl_examples/basic_examples/README.md b/pl_examples/basic_examples/README.md index fd7824140d470..05440cbad6689 100644 --- a/pl_examples/basic_examples/README.md +++ b/pl_examples/basic_examples/README.md @@ -2,47 +2,70 @@ Use these examples to test how lightning works. -#### MNIST +## MNIST Examples -Trains MNIST where the model is defined inside the `LightningModule`. +5 MNIST examples showing how to gradually convert from pure PyTorch to PyTorch Lightning. + +The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst) from pure PyTorch is optional but it might helpful to learn about it. + +#### 1 . Image Classifier with Vanilla PyTorch + +Trains a simple CNN over MNIST using vanilla PyTorch. ```bash # cpu -python simple_image_classifier.py +python mnist_examples/image_classifier_1_pytorch.py +``` -# gpus (any number) -python simple_image_classifier.py --trainer.gpus 2 +______________________________________________________________________ -# dataparallel -python simple_image_classifier.py --trainer.gpus 2 --trainer.accelerator 'dp' +#### 2. Image Classifier with LightningLite + +Trains a simple CNN over MNIST using [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst). + +```bash +# cpu / multiple gpus if available +python mnist_examples/image_classifier_2_lite.py ``` ______________________________________________________________________ -#### MNIST with DALI +Trains a simple CNN over MNIST where `LightningLite` is almost a `LightningModule`. -The MNIST example above using [NVIDIA DALI](https://developer.nvidia.com/DALI). -Requires NVIDIA DALI to be installed based on your CUDA version, see [here](https://docs.nvidia.com/deeplearning/dali/user-guide/docs/installation.html). +```bash +# cpu / multiple gpus if available +python mnist_examples/image_classifier_3_lite_to_lightning.py +``` + +______________________________________________________________________ + +#### 4. Image Classifier with LightningModule + +Trains a simple CNN over MNIST with `Lightning Trainer` and the converted `LightningModule`. ```bash -python dali_image_classifier.py +# cpu +python mnist_examples/image_classifier_4_lightning.py + +# gpus (any number) +python mnist_examples/image_classifier_4_lightning.py --trainer.gpus 2 ``` ______________________________________________________________________ -#### Image classifier +#### 5. Image Classifier with LightningModule + LightningDataModule -Generic image classifier with an arbitrary backbone (ie: a simple system) +Trains a simple CNN over MNIST with `Lightning Trainer` and the converted `LightningModule` and `LightningDataModule` ```bash # cpu -python backbone_image_classifier.py +python mnist_examples/image_classifier_5_lightning_datamodule.py # gpus (any number) -python backbone_image_classifier.py --trainer.gpus 2 +python mnist_examples/image_classifier_5_lightning_datamodule.py --trainer.gpus 2 -# dataparallel -python backbone_image_classifier.py --trainer.gpus 2 --trainer.accelerator 'dp' +# data parallel +python mnist_examples/image_classifier_5_lightning_datamodule.py --trainer.gpus 2 --trainer.accelerator 'dp' ``` ______________________________________________________________________ diff --git a/pl_examples/basic_examples/mnist_examples/README.md b/pl_examples/basic_examples/mnist_examples/README.md new file mode 100644 index 0000000000000..323273d9ff718 --- /dev/null +++ b/pl_examples/basic_examples/mnist_examples/README.md @@ -0,0 +1,67 @@ +## MNIST Examples + +5 MNIST examples showing how to gradually convert from pure PyTorch to PyTorch Lightning. + +The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst) from pure PyTorch is optional but it might helpful to learn about it. + +#### 1 . Image Classifier with Vanilla PyTorch + +Trains a simple CNN over MNIST using vanilla PyTorch. + +```bash +# cpu +python image_classifier_1_pytorch.py +``` + +______________________________________________________________________ + +#### 2. Image Classifier with LightningLite + +Trains a simple CNN over MNIST using [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst). + +```bash +# cpu / multiple gpus if available +python image_classifier_2_lite.py +``` + +______________________________________________________________________ + +#### 3. Image Classifier - Conversion Lite to Lightning + +Trains a simple CNN over MNIST where `LightningLite` is almost a `LightningModule`. + +```bash +# cpu / multiple gpus if available +python image_classifier_3_lite_to_lightning.py +``` + +______________________________________________________________________ + +#### 4. Image Classifier with LightningModule + +Trains a simple CNN over MNIST with `Lightning Trainer` and the converted `LightningModule`. + +```bash +# cpu +python mnist_examples/image_classifier_4_lightning.py + +# gpus (any number) +python mnist_examples/image_classifier_4_lightning.py --trainer.gpus 2 +``` + +______________________________________________________________________ + +#### 5. Image Classifier with LightningModule + LightningDataModule + +Trains a simple CNN over MNIST with `Lightning Trainer` and the converted `LightningModule` and `LightningDataModule` + +```bash +# cpu +python image_classifier_5_lightning_datamodule.py + +# gpus (any number) +python image_classifier_5_lightning_datamodule.py --trainer.gpus 2 + +# dataparallel +python image_classifier_5_lightning_datamodule.py --trainer.gpus 2 --trainer.accelerator 'dp' +``` diff --git a/pl_examples/lite_examples/__init__.py b/pl_examples/basic_examples/mnist_examples/__init__.py similarity index 100% rename from pl_examples/lite_examples/__init__.py rename to pl_examples/basic_examples/mnist_examples/__init__.py diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_1_pytorch.py b/pl_examples/basic_examples/mnist_examples/image_classifier_1_pytorch.py new file mode 100644 index 0000000000000..e7449473194ed --- /dev/null +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_1_pytorch.py @@ -0,0 +1,160 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +import torchvision.transforms as T +from torch.optim.lr_scheduler import StepLR + +from pl_examples.basic_examples.mnist_datamodule import MNIST + +# Credit to the PyTorch Team +# Taken from https://github.com/pytorch/examples/blob/master/mnist/main.py and slightly adapted. + + +class Net(nn.Module): + def __init__(self): + super().__init__() + self.conv1 = nn.Conv2d(1, 32, 3, 1) + self.conv2 = nn.Conv2d(32, 64, 3, 1) + self.dropout1 = nn.Dropout(0.25) + self.dropout2 = nn.Dropout(0.5) + self.fc1 = nn.Linear(9216, 128) + self.fc2 = nn.Linear(128, 10) + + def forward(self, x): + x = self.conv1(x) + x = F.relu(x) + x = self.conv2(x) + x = F.relu(x) + x = F.max_pool2d(x, 2) + x = self.dropout1(x) + x = torch.flatten(x, 1) + x = self.fc1(x) + x = F.relu(x) + x = self.dropout2(x) + x = self.fc2(x) + output = F.log_softmax(x, dim=1) + return output + + +def train(args, model, device, train_loader, optimizer, epoch): + model.train() + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(device), target.to(device) + optimizer.zero_grad() + output = model(data) + loss = F.nll_loss(output, target) + loss.backward() + optimizer.step() + if (batch_idx == 0) or ((batch_idx + 1) % args.log_interval == 0): + print( + "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format( + epoch, + batch_idx * len(data), + len(train_loader.dataset), + 100.0 * batch_idx / len(train_loader), + loss.item(), + ) + ) + if args.dry_run: + break + + +def test(args, model, device, test_loader): + model.eval() + test_loss = 0 + correct = 0 + with torch.no_grad(): + for data, target in test_loader: + data, target = data.to(device), target.to(device) + output = model(data) + test_loss += F.nll_loss(output, target, reduction="sum").item() # sum up batch loss + pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability + correct += pred.eq(target.view_as(pred)).sum().item() + if args.dry_run: + break + + test_loss /= len(test_loader.dataset) + + print( + "\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n".format( + test_loss, correct, len(test_loader.dataset), 100.0 * correct / len(test_loader.dataset) + ) + ) + + +def main(): + parser = argparse.ArgumentParser(description="PyTorch MNIST Example") + parser.add_argument( + "--batch-size", type=int, default=64, metavar="N", help="input batch size for training (default: 64)" + ) + parser.add_argument( + "--test-batch-size", type=int, default=1000, metavar="N", help="input batch size for testing (default: 1000)" + ) + parser.add_argument("--epochs", type=int, default=14, metavar="N", help="number of epochs to train (default: 14)") + parser.add_argument("--lr", type=float, default=1.0, metavar="LR", help="learning rate (default: 1.0)") + parser.add_argument("--gamma", type=float, default=0.7, metavar="M", help="Learning rate step gamma (default: 0.7)") + parser.add_argument("--no-cuda", action="store_true", default=False, help="disables CUDA training") + parser.add_argument("--dry-run", action="store_true", default=False, help="quickly check a single pass") + parser.add_argument("--seed", type=int, default=1, metavar="S", help="random seed (default: 1)") + parser.add_argument( + "--log-interval", + type=int, + default=10, + metavar="N", + help="how many batches to wait before logging training status", + ) + parser.add_argument("--save-model", action="store_true", default=False, help="For Saving the current Model") + args = parser.parse_args() + use_cuda = not args.no_cuda and torch.cuda.is_available() + + torch.manual_seed(args.seed) + + device = torch.device("cuda" if use_cuda else "cpu") + + train_kwargs = {"batch_size": args.batch_size} + test_kwargs = {"batch_size": args.test_batch_size} + if use_cuda: + cuda_kwargs = {"num_workers": 1, "pin_memory": True, "shuffle": True} + train_kwargs.update(cuda_kwargs) + test_kwargs.update(cuda_kwargs) + + transform = T.Compose([T.ToTensor(), T.Normalize((0.1307,), (0.3081,))]) + train_dataset = MNIST("./data", train=True, download=True, transform=transform) + test_dataset = MNIST("./data", train=False, transform=transform) + train_loader = torch.utils.data.DataLoader(train_dataset, **train_kwargs) + test_loader = torch.utils.data.DataLoader(test_dataset, **test_kwargs) + + model = Net().to(device) + optimizer = optim.Adadelta(model.parameters(), lr=args.lr) + + scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) + for epoch in range(1, args.epochs + 1): + train(args, model, device, train_loader, optimizer, epoch) + test(args, model, device, test_loader) + scheduler.step() + + if args.dry_run: + break + + if args.save_model: + torch.save(model.state_dict(), "mnist_cnn.pt") + + +if __name__ == "__main__": + main() diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py new file mode 100644 index 0000000000000..78677cdf33bc4 --- /dev/null +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py @@ -0,0 +1,131 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse + +import torch +import torch.nn.functional as F +import torch.optim as optim +import torchvision.transforms as T +from torch.optim.lr_scheduler import StepLR +from torchmetrics.classification import Accuracy + +from pl_examples.basic_examples.mnist_datamodule import MNIST +from pl_examples.basic_examples.mnist_examples.image_classifier_1_pytorch import Net +from pytorch_lightning import seed_everything +from pytorch_lightning.lite import LightningLite + + +def train(lite, args, model, train_loader, optimizer, epoch): + model.train() + for batch_idx, (data, target) in enumerate(train_loader): + optimizer.zero_grad() + output = model(data) + loss = F.nll_loss(output, target) + lite.backward(loss) + optimizer.step() + if (batch_idx == 0) or ((batch_idx + 1) % args.log_interval == 0): + print( + "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format( + epoch, + batch_idx * len(data), + len(train_loader.dataset), + 100.0 * batch_idx / len(train_loader), + loss.item(), + ) + ) + if args.dry_run: + break + + +def test(lite, args, model, test_loader): + model.eval() + test_loss = 0 + acc = Accuracy().to(lite.device) + with torch.no_grad(): + for data, target in test_loader: + output = model(data) + test_loss += F.nll_loss(output, target, reduction="sum").item() # sum up batch loss + acc.update(output, target) + if args.dry_run: + break + + test_loss = lite.all_gather(test_loss).sum() / len(test_loader.dataset) + + if lite.is_global_zero: + print(f"\nTest set: Average loss: {test_loss:.4f}, Accuracy: ({acc.compute():.0f}%)\n") + + +class Lite(LightningLite): + def run(self, args): + train_kwargs = {"batch_size": args.batch_size} + test_kwargs = {"batch_size": args.test_batch_size} + transform = T.Compose([T.ToTensor(), T.Normalize((0.1307,), (0.3081,))]) + train_dataset = MNIST("./data", train=True, download=True, transform=transform) + test_dataset = MNIST("./data", train=False, transform=transform) + train_loader = torch.utils.data.DataLoader(train_dataset, **train_kwargs) + test_loader = torch.utils.data.DataLoader(test_dataset, **test_kwargs) + + train_loader, test_loader = self.setup_dataloaders(train_loader, test_loader) + + model = Net() + optimizer = optim.Adadelta(model.parameters(), lr=args.lr) + + model, optimizer = self.setup(model, optimizer) + + scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) + for epoch in range(1, args.epochs + 1): + train(self, args, model, train_loader, optimizer, epoch) + test(self, args, model, test_loader) + scheduler.step() + + if args.dry_run: + break + + if args.save_model and self.is_global_zero: + torch.save(model.state_dict(), "mnist_cnn.pt") + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description="LightningLite MNIST Example") + parser.add_argument( + "--batch-size", type=int, default=64, metavar="N", help="input batch size for training (default: 64)" + ) + parser.add_argument( + "--test-batch-size", type=int, default=1000, metavar="N", help="input batch size for testing (default: 1000)" + ) + parser.add_argument("--epochs", type=int, default=14, metavar="N", help="number of epochs to train (default: 14)") + parser.add_argument("--lr", type=float, default=1.0, metavar="LR", help="learning rate (default: 1.0)") + parser.add_argument("--gamma", type=float, default=0.7, metavar="M", help="Learning rate step gamma (default: 0.7)") + parser.add_argument("--no-cuda", action="store_true", default=False, help="disables CUDA training") + parser.add_argument("--dry-run", action="store_true", default=False, help="quickly check a single pass") + parser.add_argument("--seed", type=int, default=1, metavar="S", help="random seed (default: 1)") + parser.add_argument( + "--log-interval", + type=int, + default=10, + metavar="N", + help="how many batches to wait before logging training status", + ) + parser.add_argument("--save-model", action="store_true", default=False, help="For Saving the current Model") + args = parser.parse_args() + + seed_everything(args.seed) + + if torch.cuda.is_available(): + lite_kwargs = {"accelerator": "gpu", "devices": torch.cuda.device_count()} + else: + lite_kwargs = {"accelerator": "cpu"} + + Lite(**lite_kwargs).run(args) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py new file mode 100644 index 0000000000000..223f23312586e --- /dev/null +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py @@ -0,0 +1,166 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse + +import torch +import torch.nn.functional as F +import torch.optim as optim +import torchvision.transforms as T +from torch.optim.lr_scheduler import StepLR +from torchmetrics import Accuracy + +from pl_examples.basic_examples.mnist_datamodule import MNIST +from pl_examples.basic_examples.mnist_examples.image_classifier_1_pytorch import Net +from pytorch_lightning import seed_everything +from pytorch_lightning.lite import LightningLite + + +def train(lite, args, model, train_loader, optimizer, epoch): + model.train() + for batch_idx, batch in enumerate(train_loader): + optimizer.zero_grad() + loss = lite.training_step(batch, batch_idx) + lite.backward(loss) + optimizer.step() + if (batch_idx == 0) or ((batch_idx + 1) % args.log_interval == 0): + print( + "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format( + epoch, + batch_idx * len(batch[0]), + len(train_loader.dataset), + 100.0 * batch_idx / len(train_loader), + loss.item(), + ) + ) + if args.dry_run: + break + + +def test(lite, args, model, test_loader): + model.eval() + test_loss = 0 + with torch.no_grad(): + for batch_idx, batch in enumerate(test_loader): + test_loss += lite.test_step(batch, batch_idx) + if args.dry_run: + break + + test_loss = lite.all_gather(test_loss).sum() / len(test_loader.dataset) + + if lite.is_global_zero: + print(f"\nTest set: Average loss: {test_loss:.4f}, Accuracy: ({lite.test_acc.compute():.0f}%)\n") + + +class Lite(LightningLite): + + """`Lite` is starting to look like a `LightningModule`.""" + + def run(self, hparams): + self.hparams = hparams + + self.model = Net() + [optimizer], [scheduler] = self.configure_optimizers() + model, optimizer = self.setup(self.model, optimizer) + + if self.is_global_zero: + self.prepare_data() + + train_loader, test_loader = self.setup_dataloaders(self.train_dataloader(), self.train_dataloader()) + + self.test_acc = Accuracy() + + for epoch in range(1, hparams.epochs + 1): + train(self, hparams, model, train_loader, optimizer, epoch) + test(self, hparams, model, test_loader) + scheduler.step() + + if args.dry_run: + break + + if hparams.save_model and self.is_global_zero: + torch.save(model.state_dict(), "mnist_cnn.pt") + + # Functions for the `LightningModule` conversion + + def forward(self, x): + return self.model(x) + + def training_step(self, batch, batch_idx): + x, y = batch + logits = self.forward(x) + loss = F.nll_loss(logits, y.long()) + return loss + + def test_step(self, batch, batch_idx): + x, y = batch + logits = self.forward(x) + loss = F.nll_loss(logits, y.long()) + self.test_acc(logits, y.long()) + return loss + + def configure_optimizers(self): + optimizer = optim.Adadelta(self.model.parameters(), lr=self.hparams.lr) + return [optimizer], [StepLR(optimizer, step_size=1, gamma=self.hparams.gamma)] + + # Functions for the `LightningDataModule` conversion + + @property + def transform(self): + return T.Compose([T.ToTensor(), T.Normalize((0.1307,), (0.3081,))]) + + def prepare_data(self) -> None: + MNIST("./data", download=True) + + def train_dataloader(self): + train_dataset = MNIST("./data", train=True, download=False, transform=self.transform) + return torch.utils.data.DataLoader(train_dataset, batch_size=self.hparams.batch_size) + + def test_dataloader(self): + test_dataset = MNIST("./data", train=False, download=False, transform=self.transform) + return torch.utils.data.DataLoader(test_dataset, batch_size=self.hparams.batch_size) + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description="LightningLite to LightningModule MNIST Example") + parser.add_argument( + "--batch-size", type=int, default=64, metavar="N", help="input batch size for training (default: 64)" + ) + parser.add_argument( + "--test-batch-size", type=int, default=1000, metavar="N", help="input batch size for testing (default: 1000)" + ) + parser.add_argument("--epochs", type=int, default=14, metavar="N", help="number of epochs to train (default: 14)") + parser.add_argument("--lr", type=float, default=1.0, metavar="LR", help="learning rate (default: 1.0)") + parser.add_argument("--gamma", type=float, default=0.7, metavar="M", help="Learning rate step gamma (default: 0.7)") + parser.add_argument("--no-cuda", action="store_true", default=False, help="disables CUDA training") + parser.add_argument("--dry-run", action="store_true", default=False, help="quickly check a single pass") + parser.add_argument("--seed", type=int, default=1, metavar="S", help="random seed (default: 1)") + parser.add_argument( + "--log-interval", + type=int, + default=10, + metavar="N", + help="how many batches to wait before logging training status", + ) + parser.add_argument("--save-model", action="store_true", default=False, help="For Saving the current Model") + args = parser.parse_args() + + seed_everything(args.seed) + + if torch.cuda.is_available(): + lite_kwargs = {"accelerator": "gpu", "devices": torch.cuda.device_count()} + else: + lite_kwargs = {"accelerator": "cpu"} + + Lite(**lite_kwargs).run(args) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py b/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py new file mode 100644 index 0000000000000..a414d96281b01 --- /dev/null +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py @@ -0,0 +1,86 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MNIST simple image classifier example with LightningModule. + +To run: python image_classifier_4_lightning.py --trainer.max_epochs=50 +""" +import torch +import torchvision.transforms as T +from torch.nn import functional as F +from torchmetrics.classification import Accuracy + +from pl_examples import cli_lightning_logo +from pl_examples.basic_examples.mnist_datamodule import MNIST +from pl_examples.basic_examples.mnist_examples.image_classifier_1_pytorch import Net +from pytorch_lightning import LightningModule +from pytorch_lightning.utilities.cli import LightningCLI + + +class ImageClassifier(LightningModule): + def __init__(self, model, lr=1.0, gamma=0.7, batch_size=32): + super().__init__() + self.save_hyperparameters() + self.model = model or Net() + self.test_acc = Accuracy() + + def forward(self, x): + return self.model(x) + + def training_step(self, batch, batch_idx): + x, y = batch + logits = self.forward(x) + loss = F.nll_loss(logits, y.long()) + return loss + + def test_step(self, batch, batch_idx): + x, y = batch + logits = self.forward(x) + loss = F.nll_loss(logits, y.long()) + self.test_acc(logits, y.long()) + return loss + + def test_epoch_end(self, *_) -> None: + self.log("test_acc", self.test_acc.compute()) + + def configure_optimizers(self): + optimizer = torch.optim.Adadelta(self.model.parameters(), lr=self.hparams.lr) + return [optimizer], [torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=self.hparams.gamma)] + + # Functions for the `LightningDataModule` conversion + + @property + def transform(self): + return T.Compose([T.ToTensor(), T.Normalize((0.1307,), (0.3081,))]) + + def prepare_data(self) -> None: + MNIST("./data", download=True) + + def train_dataloader(self): + train_dataset = MNIST("./data", train=True, download=False, transform=self.transform) + return torch.utils.data.DataLoader(train_dataset, batch_size=self.hparams.batch_size) + + def test_dataloader(self): + test_dataset = MNIST("./data", train=False, download=False, transform=self.transform) + return torch.utils.data.DataLoader(test_dataset, batch_size=self.hparams.batch_size) + + +def cli_main(): + cli = LightningCLI(ImageClassifier, seed_everything_default=1234, save_config_overwrite=True, run=False) + cli.trainer.fit(cli.model, datamodule=cli.datamodule) + cli.trainer.test(ckpt_path="best", datamodule=cli.datamodule) + + +if __name__ == "__main__": + cli_lightning_logo() + cli_main() diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py b/pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py new file mode 100644 index 0000000000000..fc30836b6c37b --- /dev/null +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py @@ -0,0 +1,92 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MNIST simple image classifier example with LightningModule and DataModule. + +To run: python image_classifier_5_lightning_datamodule.py --trainer.max_epochs=50 +""" +import torch +import torchvision.transforms as T +from torch.nn import functional as F +from torchmetrics.classification import Accuracy + +from pl_examples import cli_lightning_logo +from pl_examples.basic_examples.mnist_datamodule import MNIST +from pl_examples.basic_examples.mnist_examples.image_classifier_1_pytorch import Net +from pytorch_lightning import LightningDataModule, LightningModule +from pytorch_lightning.utilities.cli import LightningCLI + + +class ImageClassifier(LightningModule): + def __init__(self, model, lr=1.0, gamma=0.7, batch_size=32): + super().__init__() + self.save_hyperparameters() + self.model = model or Net() + self.test_acc = Accuracy() + + def forward(self, x): + return self.model(x) + + def training_step(self, batch, batch_idx): + x, y = batch + logits = self.forward(x) + loss = F.nll_loss(logits, y.long()) + return loss + + def test_step(self, batch, batch_idx): + x, y = batch + logits = self.forward(x) + loss = F.nll_loss(logits, y.long()) + self.test_acc(logits, y.long()) + return loss + + def test_epoch_end(self, *_) -> None: + self.log("test_acc", self.test_acc.compute()) + + def configure_optimizers(self): + optimizer = torch.optim.Adadelta(self.model.parameters(), lr=self.hparams.lr) + return [optimizer], [torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=self.hparams.gamma)] + + +class MNISTDataModule(LightningDataModule): + def __init__(self, batch_size=32): + super().__init__() + self.save_hyperparameters() + + @property + def transform(self): + return T.Compose([T.ToTensor(), T.Normalize((0.1307,), (0.3081,))]) + + def prepare_data(self) -> None: + MNIST("./data", download=True) + + def train_dataloader(self): + train_dataset = MNIST("./data", train=True, download=False, transform=self.transform) + return torch.utils.data.DataLoader(train_dataset, batch_size=self.hparams.batch_size) + + def test_dataloader(self): + test_dataset = MNIST("./data", train=False, download=False, transform=self.transform) + return torch.utils.data.DataLoader(test_dataset, batch_size=self.hparams.batch_size) + + +def cli_main(): + cli = LightningCLI( + ImageClassifier, MNISTDataModule, seed_everything_default=1234, save_config_overwrite=True, run=False + ) + cli.trainer.fit(cli.model, datamodule=cli.datamodule) + cli.trainer.test(ckpt_path="best", datamodule=cli.datamodule) + + +if __name__ == "__main__": + cli_lightning_logo() + cli_main() diff --git a/pl_examples/basic_examples/simple_image_classifier.py b/pl_examples/basic_examples/simple_image_classifier.py deleted file mode 100644 index 146f25c27c0d4..0000000000000 --- a/pl_examples/basic_examples/simple_image_classifier.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright The PyTorch Lightning team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""MNIST simple image classifier example. - -To run: python simple_image_classifier.py --trainer.max_epochs=50 -""" - -import torch -from torch.nn import functional as F - -import pytorch_lightning as pl -from pl_examples import cli_lightning_logo -from pl_examples.basic_examples.mnist_datamodule import MNISTDataModule -from pytorch_lightning.utilities.cli import LightningCLI - - -class LitClassifier(pl.LightningModule): - """ - >>> LitClassifier() # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE - LitClassifier( - (l1): Linear(...) - (l2): Linear(...) - ) - """ - - def __init__(self, hidden_dim: int = 128, learning_rate: float = 0.0001): - super().__init__() - self.save_hyperparameters() - - self.l1 = torch.nn.Linear(28 * 28, self.hparams.hidden_dim) - self.l2 = torch.nn.Linear(self.hparams.hidden_dim, 10) - - def forward(self, x): - x = x.view(x.size(0), -1) - x = torch.relu(self.l1(x)) - x = torch.relu(self.l2(x)) - return x - - def training_step(self, batch, batch_idx): - x, y = batch - y_hat = self(x) - loss = F.cross_entropy(y_hat, y) - return loss - - def validation_step(self, batch, batch_idx): - x, y = batch - y_hat = self(x) - loss = F.cross_entropy(y_hat, y) - self.log("valid_loss", loss) - - def test_step(self, batch, batch_idx): - x, y = batch - y_hat = self(x) - loss = F.cross_entropy(y_hat, y) - self.log("test_loss", loss) - - def configure_optimizers(self): - return torch.optim.Adam(self.parameters(), lr=self.hparams.learning_rate) - - -def cli_main(): - cli = LightningCLI( - LitClassifier, MNISTDataModule, seed_everything_default=1234, save_config_overwrite=True, run=False - ) - cli.trainer.fit(cli.model, datamodule=cli.datamodule) - cli.trainer.test(ckpt_path="best", datamodule=cli.datamodule) - - -if __name__ == "__main__": - cli_lightning_logo() - cli_main() diff --git a/pl_examples/domain_templates/generative_adversarial_net.py b/pl_examples/domain_templates/generative_adversarial_net.py index 48492c8ce7f04..26a6c8aa89f67 100644 --- a/pl_examples/domain_templates/generative_adversarial_net.py +++ b/pl_examples/domain_templates/generative_adversarial_net.py @@ -17,24 +17,21 @@ tensorboard --logdir default """ -import os from argparse import ArgumentParser, Namespace import numpy as np import torch import torch.nn as nn import torch.nn.functional as F -from torch.utils.data import DataLoader from pl_examples import cli_lightning_logo -from pl_examples.basic_examples.mnist_datamodule import MNIST -from pytorch_lightning.core import LightningDataModule, LightningModule +from pl_examples.basic_examples.mnist_datamodule import MNISTDataModule +from pytorch_lightning.core import LightningModule from pytorch_lightning.trainer import Trainer from pytorch_lightning.utilities.imports import _TORCHVISION_AVAILABLE if _TORCHVISION_AVAILABLE: import torchvision - from torchvision import transforms class Generator(nn.Module): @@ -212,35 +209,6 @@ def on_epoch_end(self): self.logger.experiment.add_image("generated_images", grid, self.current_epoch) -class MNISTDataModule(LightningDataModule): - """ - >>> MNISTDataModule() # doctest: +ELLIPSIS - <...generative_adversarial_net.MNISTDataModule object at ...> - """ - - def __init__(self, batch_size: int = 64, data_path: str = os.getcwd(), num_workers: int = 4): - super().__init__() - self.batch_size = batch_size - self.data_path = data_path - self.num_workers = num_workers - - self.transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize([0.5], [0.5])]) - self.dims = (1, 28, 28) - - def prepare_data(self, stage=None): - # Use this method to do things that might write to disk or that need to be done only from a single GPU - # in distributed settings. Like downloading the dataset for the first time. - MNIST(self.data_path, train=True, download=True, transform=transforms.ToTensor()) - - def setup(self, stage=None): - # There are also data operations you might want to perform on every GPU, such as applying transforms - # defined explicitly in your datamodule or assigned in init. - self.mnist_train = MNIST(self.data_path, train=True, transform=self.transform) - - def train_dataloader(self): - return DataLoader(self.mnist_train, batch_size=self.batch_size, num_workers=self.num_workers) - - def main(args: Namespace) -> None: # ------------------------ # 1 INIT LIGHTNING MODEL @@ -250,7 +218,7 @@ def main(args: Namespace) -> None: # ------------------------ # 2 INIT TRAINER # ------------------------ - # If use distubuted training PyTorch recommends to use DistributedDataParallel. + # If use distributed training PyTorch recommends to use DistributedDataParallel. # See: https://pytorch.org/docs/stable/nn.html#torch.nn.DataParallel dm = MNISTDataModule.from_argparse_args(args) trainer = Trainer.from_argparse_args(args) diff --git a/pl_examples/integration_examples/__init__.py b/pl_examples/integration_examples/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/pl_examples/basic_examples/dali_image_classifier.py b/pl_examples/integration_examples/dali_image_classifier.py similarity index 100% rename from pl_examples/basic_examples/dali_image_classifier.py rename to pl_examples/integration_examples/dali_image_classifier.py diff --git a/pl_examples/lite_examples/pytorch_2_lite_2_lightning.py b/pl_examples/lite_examples/pytorch_2_lite_2_lightning.py deleted file mode 100644 index 592d5a7ab951b..0000000000000 --- a/pl_examples/lite_examples/pytorch_2_lite_2_lightning.py +++ /dev/null @@ -1,248 +0,0 @@ -# Copyright The PyTorch Lightning team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import torch -from torch import nn -from torch.utils.data import DataLoader, Dataset - -from pytorch_lightning import seed_everything -from pytorch_lightning.lite import LightningLite - -############################################################################################# -# Section 1: PyTorch to Lightning Lite # -# # -# What is LightningLite ? # -# # -# `LightningLite` is a python class you can override to get access to Lightning # -# accelerators and scale your training, but furthermore, it is intended to be the safest # -# route to fully transition to Lightning. # -# # -# Does LightningLite requires code changes ? # -# # -# `LightningLite` code changes are minimal and this tutorial will show you how easy it # -# is to convert to `lite` using a `BoringModel`. # -# # -############################################################################################# - -############################################################################################# -# Pure PyTorch Section # -############################################################################################# - - -# 1 / 6: Implement a `BoringModel` with only one layer. -class BoringModel(nn.Module): - def __init__(self): - super().__init__() - self.layer = torch.nn.Linear(32, 2) - - def forward(self, x): - x = self.layer(x) - return torch.nn.functional.mse_loss(x, torch.ones_like(x)) - - -# 2 / 6: Implement a `configure_optimizers` taking a module and returning an optimizer. -def configure_optimizers(module: nn.Module): - return torch.optim.SGD(module.parameters(), lr=0.001) - - -# 3 / 6: Implement a simple dataset returning random data with the specified shape. -class RandomDataset(Dataset): - def __init__(self, length: int, size: int): - self.len = length - self.data = torch.randn(length, size) - - def __getitem__(self, index): - return self.data[index] - - def __len__(self): - return self.len - - -# 4 / 6: Implement the functions to create the dataloaders. -def train_dataloader(): - return DataLoader(RandomDataset(64, 32)) - - -def val_dataloader(): - return DataLoader(RandomDataset(64, 32)) - - -# 5 / 6: Our main PyTorch Loop to train our `BoringModel` on our random data. -def main(model: nn.Module, train_dataloader: DataLoader, val_dataloader: DataLoader, num_epochs: int = 10): - optimizer = configure_optimizers(model) - - for epoch in range(num_epochs): - train_losses = [] - val_losses = [] - - model.train() - for batch in train_dataloader: - optimizer.zero_grad() - loss = model(batch) - train_losses.append(loss) - loss.backward() - optimizer.step() - - model.eval() - with torch.no_grad(): - for batch in val_dataloader: - val_losses.append(model(batch)) - - train_epoch_loss = torch.stack(train_losses).mean() - val_epoch_loss = torch.stack(val_losses).mean() - - print(f"{epoch}/{num_epochs}| Train Epoch Loss: {torch.mean(train_epoch_loss)}") - print(f"{epoch}/{num_epochs}| Valid Epoch Loss: {torch.mean(val_epoch_loss)}") - - return model.state_dict() - - -# 6 / 6: Run the pure PyTorch Loop and train / validate the model. -if __name__ == "__main__": - seed_everything(42) - model = BoringModel() - pure_model_weights = main(model, train_dataloader(), val_dataloader()) - - -############################################################################################# -# Convert to LightningLite # -# # -# By converting to `LightningLite`, you get the full power of Lightning accelerators # -# while conversing your original code ! # -# To get started, you would need to `from pytorch_lightning.lite import LightningLite` # -# and override its `run` method. # -############################################################################################# - - -class LiteTrainer(LightningLite): - def run(self, model: nn.Module, train_dataloader: DataLoader, val_dataloader: DataLoader, num_epochs: int = 10): - optimizer = configure_optimizers(model) - - ################################################################################### - # You would need to call `self.setup` to wrap `model` and `optimizer`. If you # - # have multiple models (c.f GAN), call `setup` for each one of them and their # - # associated optimizers. # - model, optimizer = self.setup(model, optimizer) - ################################################################################### - - ################################################################################### - # You would need to call `self.setup_dataloaders` to prepare the dataloaders # - # in case you are running in a distributed setting. # - train_dataloader = self.setup_dataloaders(train_dataloader) - val_dataloader = self.setup_dataloaders(val_dataloader) - ################################################################################### - - for epoch in range(num_epochs): - train_losses = [] - val_losses = [] - - model.train() - for batch in train_dataloader: - optimizer.zero_grad() - loss = model(batch) - train_losses.append(loss) - ########################################################################### - # By calling `self.backward` directly, `LightningLite` will automate # - # precision and distributions. # - self.backward(loss) - ########################################################################### - optimizer.step() - - model.eval() - with torch.no_grad(): - for batch in val_dataloader: - val_losses.append(model(batch)) - - train_epoch_loss = torch.stack(train_losses).mean() - val_epoch_loss = torch.stack(val_losses).mean() - - ################################################################################ - # Optional: Utility to print only on rank 0 (when using distributed setting) # - self.print(f"{epoch}/{num_epochs}| Train Epoch Loss: {train_epoch_loss}") - self.print(f"{epoch}/{num_epochs}| Valid Epoch Loss: {val_epoch_loss}") - ################################################################################ - - -if __name__ == "__main__": - seed_everything(42) - lite_model = BoringModel() - lite = LiteTrainer() - lite.run(lite_model, train_dataloader(), val_dataloader()) - - ############################################################################################# - # Assert the weights are the same # - ############################################################################################# - - for pure_w, lite_w in zip(pure_model_weights.values(), lite_model.state_dict().values()): - torch.equal(pure_w, lite_w) - - -############################################################################################# -# Convert to Lightning # -# # -# By converting to Lightning, not-only your research code becomes inter-operable # -# (can easily be shared), but you get access to hundreds of extra features to make your # -# research faster. # -# Check `Facebook` blogpost on how `Lightning` enabled their research to scale at scale # -# On https://ai.facebook.com/blog # -# /reengineering-facebook-ais-deep-learning-platforms-for-interoperability/ # -############################################################################################# - -from pytorch_lightning import LightningDataModule, LightningModule, Trainer # noqa E402 - - -class LightningBoringModel(LightningModule): - def __init__(self): - super().__init__() - self.layer = torch.nn.Linear(32, 2) - - def forward(self, x): - x = self.layer(x) - return torch.nn.functional.mse_loss(x, torch.ones_like(x)) - - # LightningModule hooks - def training_step(self, batch, batch_idx): - x = self.forward(batch) - self.log("train_loss", x) - return x - - def validation_step(self, batch, batch_idx): - x = self.forward(batch) - self.log("val_loss", x) - return x - - def configure_optimizers(self): - return configure_optimizers(self) - - -class BoringDataModule(LightningDataModule): - def train_dataloader(self): - return train_dataloader() - - def val_dataloader(self): - return val_dataloader() - - -if __name__ == "__main__": - seed_everything(42) - lightning_module = LightningBoringModel() - datamodule = BoringDataModule() - trainer = Trainer(max_epochs=10) - trainer.fit(lightning_module, datamodule) - - ############################################################################################# - # Assert the weights are the same # - ############################################################################################# - - for pure_w, lite_w in zip(pure_model_weights.values(), lightning_module.state_dict().values()): - torch.equal(pure_w, lite_w) diff --git a/pl_examples/loop_examples/mnist_lite.py b/pl_examples/loop_examples/mnist_lite.py new file mode 100644 index 0000000000000..738964a56f6dc --- /dev/null +++ b/pl_examples/loop_examples/mnist_lite.py @@ -0,0 +1,189 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +from typing import Any + +import torch +import torch.nn.functional as F +import torch.optim as optim +import torchvision.transforms as T +from torch.optim.lr_scheduler import StepLR +from torchmetrics import Accuracy + +from pl_examples.basic_examples.mnist_datamodule import MNIST +from pl_examples.basic_examples.mnist_examples.image_classifier_1_pytorch import Net +from pytorch_lightning import seed_everything +from pytorch_lightning.lite import LightningLite +from pytorch_lightning.loops import Loop + + +class TrainLoop(Loop): + def __init__(self, lite, args, model, optimizer, scheduler, dataloader): + super().__init__() + self.lite = lite + self.args = args + self.model = model + self.optimizer = optimizer + self.scheduler = scheduler + self.dataloader = dataloader + + @property + def done(self) -> bool: + return False + + def reset(self): + self.dataloader_iter = enumerate(self.dataloader) + + def advance(self, epoch) -> None: + batch_idx, (data, target) = next(self.dataloader_iter) + self.optimizer.zero_grad() + output = self.model(data) + loss = F.nll_loss(output, target) + self.lite.backward(loss) + self.optimizer.zero_grad() + + if (batch_idx == 0) or ((batch_idx + 1) % self.args.log_interval == 0): + print( + "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format( + epoch, + batch_idx * len(self.dataloader), + len(self.dataloader.dataset), + 100.0 * batch_idx / len(self.dataloader), + loss.item(), + ) + ) + + if self.args.dry_run: + raise StopIteration + + def on_run_end(self): + self.scheduler.step() + self.dataloader_iter = None + + +class TestLoop(Loop): + def __init__(self, lite, args, model, dataloader): + super().__init__() + self.lite = lite + self.args = args + self.model = model + self.dataloader = dataloader + self.accuracy = Accuracy() + + @property + def done(self) -> bool: + return False + + def reset(self): + self.dataloader_iter = enumerate(self.dataloader) + self.test_loss = 0 + self.accuracy.reset() + + def advance(self) -> None: + _, (data, target) = next(self.dataloader_iter) + output = self.model(data) + self.test_loss += F.nll_loss(output, target) + self.accuracy(output, target) + + if self.args.dry_run: + raise StopIteration + + def on_run_end(self): + test_loss = self.lite.all_gather(self.test_loss).sum() / len(self.dataloader.dataset) + + if self.lite.is_global_zero: + print(f"\nTest set: Average loss: {test_loss:.4f}, Accuracy: ({self.accuracy.compute():.0f}%)\n") + + +class MainLoop(Loop): + def __init__(self, lite, args, model, optimizer, scheduler, train_loader, test_loader): + super().__init__() + self.lite = lite + self.args = args + self.epoch = 0 + self.train_loop = TrainLoop(self.lite, self.args, model, optimizer, scheduler, train_loader) + self.test_loop = TestLoop(self.lite, self.args, model, test_loader) + + @property + def done(self) -> bool: + return self.epoch >= self.args.epochs + + def reset(self): + pass + + def advance(self, *args: Any, **kwargs: Any) -> None: + self.train_loop.run(self.epoch) + self.test_loop.run() + + if self.args.dry_run: + raise StopIteration + + self.epoch += 1 + self.lite.val_acc.reset() + + +class Lite(LightningLite): + def run(self, hparams): + transform = T.Compose([T.ToTensor(), T.Normalize((0.1307,), (0.3081,))]) + train_dataset = MNIST("./data", train=True, download=True, transform=transform) + test_dataset = MNIST("./data", train=False, transform=transform) + train_loader = torch.utils.data.DataLoader(train_dataset, hparams.batch_size) + test_loader = torch.utils.data.DataLoader(test_dataset, hparams.test_batch_size) + + train_loader, test_loader = self.setup_dataloaders(train_loader, test_loader) + + model = Net() + optimizer = optim.Adadelta(model.parameters(), lr=args.lr) + + model, optimizer = self.setup(model, optimizer) + scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) + + MainLoop(self, args, model, optimizer, scheduler, train_loader, test_loader).run() + + if args.save_model and self.is_global_zero: + torch.save(model.state_dict(), "mnist_cnn.pt") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="LightningLite MNIST Example with Lightning Loops.") + parser.add_argument( + "--batch-size", type=int, default=64, metavar="N", help="input batch size for training (default: 64)" + ) + parser.add_argument( + "--test-batch-size", type=int, default=1000, metavar="N", help="input batch size for testing (default: 1000)" + ) + parser.add_argument("--epochs", type=int, default=14, metavar="N", help="number of epochs to train (default: 14)") + parser.add_argument("--lr", type=float, default=1.0, metavar="LR", help="learning rate (default: 1.0)") + parser.add_argument("--gamma", type=float, default=0.7, metavar="M", help="Learning rate step gamma (default: 0.7)") + parser.add_argument("--no-cuda", action="store_true", default=False, help="disables CUDA training") + parser.add_argument("--dry-run", action="store_true", default=False, help="quickly check a single pass") + parser.add_argument("--seed", type=int, default=1, metavar="S", help="random seed (default: 1)") + parser.add_argument( + "--log-interval", + type=int, + default=10, + metavar="N", + help="how many batches to wait before logging training status", + ) + parser.add_argument("--save-model", action="store_true", default=False, help="For Saving the current Model") + args = parser.parse_args() + + seed_everything(args.seed) + + if torch.cuda.is_available(): + lite_kwargs = {"accelerator": "gpu", "devices": torch.cuda.device_count()} + else: + lite_kwargs = {"accelerator": "cpu"} + + Lite(**lite_kwargs).run(args) diff --git a/pl_examples/run_examples.sh b/pl_examples/run_examples.sh index 7555e472d24e2..e6b0c6bef1170 100755 --- a/pl_examples/run_examples.sh +++ b/pl_examples/run_examples.sh @@ -11,6 +11,23 @@ args=" --trainer.limit_predict_batches=2 " -python "${dir_path}/basic_examples/simple_image_classifier.py" ${args} "$@" python "${dir_path}/basic_examples/backbone_image_classifier.py" ${args} "$@" python "${dir_path}/basic_examples/autoencoder.py" ${args} "$@" + + +args=" + --trainer.max_epochs=1 + --trainer.limit_train_batches=2 + --trainer.limit_val_batches=2 + --trainer.limit_test_batches=2 + --trainer.limit_predict_batches=2 +" + +python "${dir_path}/basic_examples/mnist_examples/image_classifier_4_lightning.py" ${args} "$@" +python "${dir_path}/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py" ${args} "$@" + +args="--dry-run" +python "${dir_path}/basic_examples/mnist_examples/image_classifier_1_pytorch.py" ${args} "$@" +python "${dir_path}/basic_examples/mnist_examples/image_classifier_2_lite.py" ${args} "$@" +python "${dir_path}/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py" ${args} "$@" +python "${dir_path}/loop_examples/mnist_lite.py" ${args} "$@" diff --git a/pl_examples/test_examples.py b/pl_examples/test_examples.py index b0b451692e4b6..19d09836ef34c 100644 --- a/pl_examples/test_examples.py +++ b/pl_examples/test_examples.py @@ -34,7 +34,7 @@ @RunIf(min_gpus=1, skip_windows=True) @pytest.mark.parametrize("cli_args", [ARGS_GPU]) def test_examples_mnist_dali(tmpdir, cli_args): - from pl_examples.basic_examples.dali_image_classifier import cli_main + from pl_examples.integration_examples.dali_image_classifier import cli_main # update the temp dir cli_args = cli_args % {"tmpdir": tmpdir} From b914b745f99e8082616f72fb2c08f7a9602045c5 Mon Sep 17 00:00:00 2001 From: tchaton Date: Tue, 26 Oct 2021 20:18:48 +0100 Subject: [PATCH 222/331] update --- .../mnist_examples/image_classifier_4_lightning.py | 2 +- pl_examples/loop_examples/kfold.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py b/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py index a414d96281b01..6b73bfa20bf8f 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py @@ -28,7 +28,7 @@ class ImageClassifier(LightningModule): - def __init__(self, model, lr=1.0, gamma=0.7, batch_size=32): + def __init__(self, model=None, lr=1.0, gamma=0.7, batch_size=32): super().__init__() self.save_hyperparameters() self.model = model or Net() diff --git a/pl_examples/loop_examples/kfold.py b/pl_examples/loop_examples/kfold.py index 632734b30137c..878ebc73054b7 100644 --- a/pl_examples/loop_examples/kfold.py +++ b/pl_examples/loop_examples/kfold.py @@ -27,7 +27,7 @@ from pl_examples import _DATASETS_PATH from pl_examples.basic_examples.mnist_datamodule import MNIST -from pl_examples.basic_examples.simple_image_classifier import LitClassifier +from pl_examples.basic_examples.mnist_examples.image_classifier_4_lightning import ImageClassifier from pytorch_lightning import LightningDataModule, seed_everything, Trainer from pytorch_lightning.core.lightning import LightningModule from pytorch_lightning.loops.base import Loop @@ -241,7 +241,7 @@ def __getattr__(self, key) -> Any: ############################################################################################# if __name__ == "__main__": - model = LitClassifier() + model = ImageClassifier() datamodule = MNISTKFoldDataModule() trainer = Trainer( max_epochs=10, From 2969f6838f313b7ee2a71eb0734073f443b05356 Mon Sep 17 00:00:00 2001 From: tchaton Date: Tue, 26 Oct 2021 21:10:58 +0100 Subject: [PATCH 223/331] update --- .gitignore | 1 + grid_generated_0.png | Bin 7000 -> 0 bytes grid_ori_0.png | Bin 1219 -> 0 bytes 3 files changed, 1 insertion(+) delete mode 100644 grid_generated_0.png delete mode 100644 grid_ori_0.png diff --git a/.gitignore b/.gitignore index 4229c050e9b7f..29d9f6c87b623 100644 --- a/.gitignore +++ b/.gitignore @@ -139,6 +139,7 @@ mnist/ legacy/checkpoints/ *.gz *ubyte +*0.png # pl tests ml-runs/ diff --git a/grid_generated_0.png b/grid_generated_0.png deleted file mode 100644 index 77820f68637fd9ace9e72d03a3af6e1cd155db03..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7000 zcmW+*2{@G9+ne9+ubW){ho`Z~u~k7qAuuTDfm%|F zgOLnH73n(pJt-z8hL8Quz^4c2&!1-vJAgn4a;z;ih-wC`xxhvch}mA+83_sd<`-w- z!%>luhq<}koSZ^yM^usXbA2TaMbw7T@yW?UtgQOiuU|ku4G%vUQG3JO+}zYO0xPa^ zuh(mR-YQAoQ8}?DKi|5tx7C` z60)-t!I)y93v#f==$PKTSyx~0@8@S#6 z&9i6E?rsn5ySln^9XS%PJaOH@VPd*7`^=d$q8I&F*5)jbT$Apq-7d-$J(?L>UU0TS zG^E0Pgo%ae92IX@*xlSb1ABpMUAWWq=3;=x#ft}-YTDWe`ubc*E>jr_0>Q$<@)#@5 z9R2##I`tG7UPlNrM`Hv_N(qlu8F#W z!tAOVYbWtlt zJzy=Tq{PSHf9t8R`g>!dx3_Y*_R`v#G8M0v?s@m_;fPvENy)PE@+hqMv)3~Kw(U;b40j`LJ!Af;Bnu9BX8Nnke zD#yd7ckP;-Whq<^JSuL3crl(IKYrjOo+l(EK;sZ;-sMR+2`MS5)OIJ57UKE-V7z1N zn+INAu$%%lx^@K34?Axs(%ReG{VT0ee8JmuC7GF-!*_1pyeS}O%@>KickdpN_9-nb zta%(0vR6@5q=>;(Wo2bmS68Q|GD-M&dq0hi?w<25kWV4JHu_7kIE*T;bF6tH@kA>%Ju6pSaB#(rcT=3yLUH-!?*#x zY^nlJAd!|PCO`Z70{)EA*PABC=y5K}0S*p2`CZw@Sp(%hCB?;NxhAHjrfKbuwZt0j z6-Zj$6@K^O76^oA+1WEsj-SDb9~Fwp%g#P>_%NAnFBNwBs^t09W7(%Qz{oE`C* za>i7UXL1rd>U_h8P4CeC$2PH+(ELE>aV9v*pBRoFoF)^s+U=4V%zqF{8C zfSiu5Zb?arE*1+lqT=7RJ3%vj-gxrliPneO5vY?Q0&(={QJe%6H%?v`>6j*5viIHRuHELc4yObg?^Erzc5l8(c(~~mritY`fMwYmx^nh1OO3Hb+>JKbso^u+S(dAl7pLj zf4Noj7%y+MP8wdSsBdm+Y6^jH8LGOdsCZ1G_H%4XO3IfnU+8ptS=j{}o1w);cNAZ5 zPY*RAVSTnoF-d=Na?%F~Wn;rPgW)&bA)RTwwY56)?wu3@F+DvkE-tREt^MfHBbdq> zTUGB_L6{qQZEbDe$9r*fdqAk*;9yunIWadsKelr;Y9K1N$%iCKAi&ahfo~YuswY_}^fV$33kV6^nbZ5f@ z$+iCG-7}m76A4};e(Drd)C>(kxVOSE0DOQREpQ!Bi^M+EPJz>jXa)mN| zgC!3poK0ZE7B(1b|Erlfgg_)WA_wO^`G?I$DbLWoa>C>=P{wT?W zX3!|b1qFm#w>IkpY+!z9&RtKisf&w?BO@aqNls~K1a7a*`B#dNwBTObzEmVgD+!&I zka$~A@T_*ExTFM55`|3#7#N?J2wv-P4JPuU<9Ud*X17 z&`zNLfW4`B*6|DH&fO3vSM<3G#eBTe`~p^lX%*n-*EcY*Dfhm|cy~Qf`JVsMuPaVY z?{N}pL0cqe|eW%6cbOkHxXhJ)9?rQ3f3Jz!_|Yj=D7 zs>E}=)bjeeX3&G#9}8_Zh*=J%F{^$l)f!5=;r3e8@WP1 zo$TxDi$;#JE;RaKQw&+hs=KxB#@g-X4E!Ei;?`e|5$ZV-xj$C&yr z3)(4J)#HncvPJGsK(08tv5AQ{s2;0wZ;-%o^s)Ya%W!RvkuTx*`aWKj;04-xPSH!p zOP!IG6*+nGzAAEeclX4J6Cj#((tvez(+!t=D=tTHN%OH6)O{XV1f&NAm2XoWT>FR< zD}L_$`I{LGs|sI_&a9iKPMrdv5cBi*H_u?)v9}K$H3J5{bcw{60{MhGdzNONJ1{T+ z%2*+>ML$CkemB1>WPQNr!Gj0oJ`2_*uEG>Ot0IROyi~d0^2Cd?7A#DcEG_#3UNMDh z!weW(SX@gX5KzYY`rqN+0CCOTJBxBOs zodPjK495M=mpQQl_`dtb%hvj zqyqI@S64#qh;3o_LiK(ykXcYrkl3c%@*1>NR~LwAe+6oVUEP!Wlixx81Dp6~fF0=V z?ai;+%KGr(2dEhr7qq-QPrteXRYOCABdi9_)8@1SP1o+k(#(n9{`Bb+3~;l(f_qm} zb8}6^i?}$(z(C80JCR7_62$~=ERL+KthBe^kfG>hFw#;}!2^NyILm(xXp4ztW@d&s z?&0md_A}Q4fza?>{I9BN5b)m5uh1+fIe04)*xyB&KkEF23nz{pV{~*ByA4&voi=!g ziP6u1neWUtJr?B0?u7CeqU-QwSTs+fTK*!c>11y4u>boE#T;nPH|^aIm#( z!v0#H+kR*K(vk;YotN7BTpu9B2fI4+3!q9+5FIU0a;QAz@x30?QhM^QO)C7##x|? z`k^){DYwBmuP+Q~2JiUl>K^iC3dM?pD%VZ{(+u702i1Tf2TI-9-G%0bThi8M!UgYc z(SYjB&^Ccf2a#MLx=gY=!IiI5hX%F?YXtd+e z7cO2*(oPv(^5u`B`L9l|H#z$HmVWwVp4|y8ssob)rdvVbheOd_nmHV3(GFy&ib})e zatgu8+uPgCO|JV$$-TaJhM6o(B$MoL?UbdZCGC{9DKC+W{(K08yU3wKzaKw-4D-dy z#=;(+YAkb@Mccrj6=s=*=>QAU)cm|oT04lYxw$#Klo6VLp}X+1IQh4iS{u>P!2vY3 zscDy6)h6qTxr%E)H2UJ&8W@(G++0p>?vlPawWOzd>0F$gaZk8Y27{Rkqn#p; zM(4mCz_hZ4T|)5@m4pNY5`avgI^SDc!ypv|=;Yz*Wlkk7~n zbOETt7#+aCsF)bQ&wF?89*x8TlDOZ$59J3_0!so$glqr9!&cDsvc*9;v9h$(7e)pu zcQIhCqR0&T#9g(&t4n~S_3hiYjg1YE7BK!WeK3Y7ik<@1g=W6Hv-2363NuE=K_7Uj zC0)IGl}Pg^60-^l7MDgFK&b%kJO9k7u`677ko@c!4f(K$7AXD-U2pFmgO2kWLcM@TbypbR_EiEkiKv4GgBZY+6p9A`QK6(NV zsY)r#B?#rc<>dX@xw(VapMjjwPC>ZU*eje9#Dm8($p!$?Nm~LnXz)g(EO7G==s>7? zWktnbU1eiw3HswGAAz&5cmWHY)aj4yShHD zvpcV(RN=dn3Z_C*@(wh&tLr?h0la8sWd)$tL8O5K0Yrz%)fFehBP$ys;1CqF3jqtj zH%bL)3j`Q^6gFuhLjlqU$ygP<>kmv~Xvmw~3Mw1)4qjiMpN&N{WT%jT8uKjEjTNbU zFvA~-wUZZwbU^~rv7Dl!`;eLeqM2l`&h`}XM%shs_;2!QOiZQMlp(lj$Q}X%cUD%4 z_0qF)b3wZU<5+xjA>$+t95_IsP<-b5J?`I+ij4(B>u+ELh5-;tH?2J@JNr^%a(uj! zqGFNfcoXnNxc2R$p47Co0Du1+6K7z%7@ai3Ot3>5BHWQ!Su`5@>h*>D6JYi275o4J z5s1(yf`WpO8tSVh4fghKEsoSiM@NHZ3D+ja$J;9j!Bq0|^QWe!zIyelz|6(pUyY>2 zcI3!OVCkKWLdI0F8JfXh96x^CCU_Hk^3Ja)IY`VjH-?!J2vC-gAI}0xf#_JSaDkL9EtQaztOR-XnZ#V;Iv~y)32z{g zNRX)62A_zj2VLLP1keFPI1ahsqww&`8D}_PL#!zDJ7h-YxyqWFq9iRac>fXJ!|7xb zVCqA_SsG>NLp%#}ps871Q*&uil^7iShHk$HCx7wz%Z4kyNibn3;swX*5K+@ zYEn{5Vbga zzg;2Lq1*fK?fmtt+gn@jji>*NH$&{n$$A;~g+M@4V#Nz{b8~ZYW>;srAOaHO3Ei?Q z>HGiK$5@79WMt%D84U7KSXdZBZck4LgFrukkB^U!hnxnS{)oGEr9X(*@2@W~*F#_m z$tIj)rm6}PNADOKS{NS>ud)4)fQrO=`uH5W_;q%6c5inZ=Fder5!l=?Q!4JRySu!i z;(af#c&zyN*jT9JUBH(87C@^6GWjSI$$fdE1*Au=Ng|cn^YiCV2IGGveM$OYPvUVB zAZ|c{G>ldTZ38bwgooci@ey81@*dt8L;%%^@x8 zsWyy)EQ4@2N#D)c85oK|q=Cwds)Mc$*%=iD4TF;amd@_f6DJRn1q1{j&;ywf6jVt- zDG9|uQU}Zg*a6`f0ma8e`UZN2h2?8Q!?JG$j0Fo*p-Yc~JgvF884LxiBNFSQiiAWK zcyEDXprwWIy$17KXtwib{2G!g0a{p4knP3%=JF)e-NM`)Qk!UOJ3E8R(vUWZlN*UN$Qf)4&En`@6W^$iaU5g&7x+n9MFnhS0)^sH)(_GantsVw zqs;PY?FeMm?>dPvxDX1Ddv0!Qs3z${D3U<8hlA&L$(=n5%g1P^bh~`i1StZR3ib`O zFzjn`^Cl%BVRB(XR7j||tLt34?)uQ^(gtjDa(w)@{7bNN`DTR=LDlH$R{O8Avjp$1b`fd4t*zXz uIqK`{L8ry&Fb^HraocsAVGoz2@>Rw zVrFK>>-APwR|i_y*4E~5I9@+ua&l6sRDOrs-vNiiF&d2kfb;Y7fC`(Mn!uOXY___( zy6t9Qd{0>{R(W~(;Nal% z^Yg^S1a5mo_zPcIYHI5C_BNGDeSCZ@Dk@rATf<&Ur_*0vUS3~cKYSV;9lg4`DiVpN zrlu+?DzFIh`Fx2)l97?|S%Z*}5V>5AAV_z2cW-YmcB_GrnVAWPy4h?dQI^eSZ)|J; z0PJ>qPEHQSs*#Zq&l>;$k00-T9v&WI5yUfnI2=xYe?Po78jUyhl9uC z<>lqY#l}`UcR}xiBYGqv9YtW6J94JB?$xq9*<|WTIqCpWMt&o+1bj< zidZbx>2$cQ{yQ@>GjN#b_4=@|FdPbng@p!#;p*zjY&IVq9a$_E|An@`zMgc?5{bm) z@jN^{2!%q@#Su1}y}!Q?-4_=Z;}f~NyZd7I+1c62&(9~m0m4%`x z002eN($Z4gI$SPS{6=D8qVK*?D3r-$zQ}fWcgb*#Aa5fD<3bw%QBhG#OH0u1?CdPb z>VAZVhDxQ<>+5Tm%eB0`92*;pLxe`7<>lq|^z?Lhcel53)J(EVkKf005Org|TE{U|@K7xS*gwtJSWruYa~fqbMp62uQFP z5fO26a`Jt2w~C zXMTQOC=~wq@q+|&;o;$ni;FOkwY4?>Q+c4&*4DPPv~amxERC5==KcLW0N~)@0C$fN zxX?~dPZQMnh3^&a>FFswJsr1>TrPLJ-2i~&`1N+Ae>EG#Ve&JHOl zDFoG;nwoBJZ{eEf3wmg1C@n1wrV|zx#$vHpELK870!#z|(AwHcuwNJo03bIvm!K3v zqtUEZD|Dt%D2U3vp-?Es#>P+-g|o@G&8t+Zv$L~z2i&QtDdIgc7>v`?Q*uISXlQ`W z_V)JJY&LOo@2IJ%866#UI-UMwhx9}sCX>nQ^-85uqC)BE>F@|k;xPG;s;VlDM&lpH z+}s@TS>q3!jAU*={{svL1IAGP{ta&aBlPw4Aqc`|vuQNi*DdjXqy6L1>-E#q)0dZ* hpM41p5+n$5egYD7Ldw>)hDQJZ002ovPDHLkV1kjyQ?md7 From 57f82e9884df779615f3fe03911835dc7a0594db Mon Sep 17 00:00:00 2001 From: tchaton Date: Tue, 26 Oct 2021 21:12:07 +0100 Subject: [PATCH 224/331] update --- tests/special_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/special_tests.sh b/tests/special_tests.sh index 1346cea295d54..edb6f2980c698 100755 --- a/tests/special_tests.sh +++ b/tests/special_tests.sh @@ -88,8 +88,8 @@ fi # test that a user can manually launch individual processes args="--trainer.gpus 2 --trainer.accelerator ddp --trainer.max_epochs=1 --trainer.limit_train_batches=1 --trainer.limit_val_batches=1 --trainer.limit_test_batches=1" -MASTER_ADDR="localhost" MASTER_PORT=1234 LOCAL_RANK=1 python pl_examples/basic_examples/simple_image_classifier.py ${args} & -MASTER_ADDR="localhost" MASTER_PORT=1234 LOCAL_RANK=0 python pl_examples/basic_examples/simple_image_classifier.py ${args} +MASTER_ADDR="localhost" MASTER_PORT=1234 LOCAL_RANK=1 python pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py ${args} & +MASTER_ADDR="localhost" MASTER_PORT=1234 LOCAL_RANK=0 python pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py ${args} report+="Ran\tmanual ddp launch test\n" # echo test report From bc082a97394000700043333d744d45c47429502a Mon Sep 17 00:00:00 2001 From: tchaton Date: Tue, 26 Oct 2021 21:14:27 +0100 Subject: [PATCH 225/331] resolve doctest --- docs/source/starter/lightning_lite.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/starter/lightning_lite.rst b/docs/source/starter/lightning_lite.rst index ed60ec63c284e..788af5eb8abaf 100644 --- a/docs/source/starter/lightning_lite.rst +++ b/docs/source/starter/lightning_lite.rst @@ -404,7 +404,7 @@ num_nodes Number of cluster nodes for distributed operation. -.. testcode:: +.. code-block:: python # Default used by Lite lite = Lite(num_nodes=1) From ef6b591d6ae9921174a428b4a9d5a429f6659d81 Mon Sep 17 00:00:00 2001 From: tchaton Date: Tue, 26 Oct 2021 21:15:57 +0100 Subject: [PATCH 226/331] resolve mypy --- pytorch_lightning/plugins/precision/precision_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/plugins/precision/precision_plugin.py b/pytorch_lightning/plugins/precision/precision_plugin.py index ed60df7dd971a..62e1e33232480 100644 --- a/pytorch_lightning/plugins/precision/precision_plugin.py +++ b/pytorch_lightning/plugins/precision/precision_plugin.py @@ -103,7 +103,7 @@ def post_backward(self, model: "pl.LightningModule", closure_loss: Tensor) -> Te model.trainer.call_hook("on_after_backward") return closure_loss - def _run_backward(self, tensor: Tensor, model: Module, *args: Any, **kwargs: Any) -> None: + def _run_backward(self, tensor: Tensor, model: Optional[Module], *args: Any, **kwargs: Any) -> None: """Lightning-independent backward logic. Currently only used by Lightning Lite. Subject to further refactors. From c90aff58d1863475c90ca4d981b5949daeacb21e Mon Sep 17 00:00:00 2001 From: tchaton Date: Tue, 26 Oct 2021 21:26:49 +0100 Subject: [PATCH 227/331] update --- pytorch_lightning/lite/lite.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index f94809afe626f..170cf226b3b5a 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -277,15 +277,15 @@ def cast(self) -> Generator[None, None, None]: @overload def to_device(self, obj: nn.Module) -> nn.Module: - pass + ... @overload def to_device(self, obj: Tensor) -> Tensor: - pass + ... @overload def to_device(self, obj: Any) -> Any: - pass + ... def to_device(self, obj: Union[nn.Module, Tensor, Any]) -> Union[nn.Module, Tensor, Any]: """Move a :class:`torch.nn.Module` or a collection of tensors to the current device, if it is not already From fd8660c10102dada28d998736173bc34df892916 Mon Sep 17 00:00:00 2001 From: tchaton Date: Tue, 26 Oct 2021 21:35:30 +0100 Subject: [PATCH 228/331] switch to Any --- pytorch_lightning/lite/lite.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 170cf226b3b5a..18f069bf945e1 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -16,7 +16,7 @@ from contextlib import contextmanager from functools import partial from pathlib import Path -from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, overload, Sequence, Tuple, Union +from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, Sequence, Tuple, Union import torch import torch.nn as nn @@ -275,19 +275,7 @@ def cast(self) -> Generator[None, None, None]: with self._precision_plugin.forward_context(): yield - @overload - def to_device(self, obj: nn.Module) -> nn.Module: - ... - - @overload - def to_device(self, obj: Tensor) -> Tensor: - ... - - @overload def to_device(self, obj: Any) -> Any: - ... - - def to_device(self, obj: Union[nn.Module, Tensor, Any]) -> Union[nn.Module, Tensor, Any]: """Move a :class:`torch.nn.Module` or a collection of tensors to the current device, if it is not already on that device. From 4f8e3a58b8441be4959584c02eda2e22783db01a Mon Sep 17 00:00:00 2001 From: tchaton Date: Tue, 26 Oct 2021 21:38:32 +0100 Subject: [PATCH 229/331] update --- pytorch_lightning/lite/wrappers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 34455f4ca091c..b09070641f73f 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -142,7 +142,7 @@ def __init__(self, device: Optional[torch.device] = None, **dl_kwargs: Any) -> N def device(self) -> Optional[torch.device]: return self._device - def __iter__(self) -> Union[Iterator[Any], Generator[Any, None, None]]: # type: ignore[override] + def __iter__(self) -> Union[Iterator[Any], Generator[Any, None, None]]: # type: ignore iterator = super().__iter__() if self._device is None: return iterator From e5fd5b62ab304b0055ac0292b16925a5e63bf763 Mon Sep 17 00:00:00 2001 From: tchaton Date: Tue, 26 Oct 2021 21:41:06 +0100 Subject: [PATCH 230/331] update --- pytorch_lightning/lite/wrappers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index b09070641f73f..e1d16ca8a3384 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -142,7 +142,7 @@ def __init__(self, device: Optional[torch.device] = None, **dl_kwargs: Any) -> N def device(self) -> Optional[torch.device]: return self._device - def __iter__(self) -> Union[Iterator[Any], Generator[Any, None, None]]: # type: ignore + def __iter__(self) -> Union[Iterator[Any], Generator[Any, None, None]]: iterator = super().__iter__() if self._device is None: return iterator From bca53f66b37583986ad44d9751e9e55f1445d1ee Mon Sep 17 00:00:00 2001 From: tchaton Date: Tue, 26 Oct 2021 21:59:52 +0100 Subject: [PATCH 231/331] update --- tests/helpers/pipelines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/pipelines.py b/tests/helpers/pipelines.py index 3e5066d708da0..f8659475a330f 100644 --- a/tests/helpers/pipelines.py +++ b/tests/helpers/pipelines.py @@ -67,7 +67,7 @@ def run_model_test( assert trainer.state.finished, f"Training failed with {trainer.state}" # Check that the model is actually changed post-training change_ratio = torch.norm(initial_values - post_train_values) - assert change_ratio > 0.1, f"the model is changed of {change_ratio}" + assert change_ratio > 0.01, f"the model is changed of {change_ratio}" # test model loading pretrained_model = load_model_from_checkpoint(logger, trainer.checkpoint_callback.best_model_path, type(model)) From e0cee6a3b5da9218d08d651fddfdfdb089425c4a Mon Sep 17 00:00:00 2001 From: tchaton Date: Tue, 26 Oct 2021 22:32:56 +0100 Subject: [PATCH 232/331] resolve bugs --- grid_generated_1.png | Bin 0 -> 7000 bytes pl_examples/run_examples.sh | 8 ++++---- 2 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 grid_generated_1.png diff --git a/grid_generated_1.png b/grid_generated_1.png new file mode 100644 index 0000000000000000000000000000000000000000..77820f68637fd9ace9e72d03a3af6e1cd155db03 GIT binary patch literal 7000 zcmW+*2{@G9+ne9+ubW){ho`Z~u~k7qAuuTDfm%|F zgOLnH73n(pJt-z8hL8Quz^4c2&!1-vJAgn4a;z;ih-wC`xxhvch}mA+83_sd<`-w- z!%>luhq<}koSZ^yM^usXbA2TaMbw7T@yW?UtgQOiuU|ku4G%vUQG3JO+}zYO0xPa^ zuh(mR-YQAoQ8}?DKi|5tx7C` z60)-t!I)y93v#f==$PKTSyx~0@8@S#6 z&9i6E?rsn5ySln^9XS%PJaOH@VPd*7`^=d$q8I&F*5)jbT$Apq-7d-$J(?L>UU0TS zG^E0Pgo%ae92IX@*xlSb1ABpMUAWWq=3;=x#ft}-YTDWe`ubc*E>jr_0>Q$<@)#@5 z9R2##I`tG7UPlNrM`Hv_N(qlu8F#W z!tAOVYbWtlt zJzy=Tq{PSHf9t8R`g>!dx3_Y*_R`v#G8M0v?s@m_;fPvENy)PE@+hqMv)3~Kw(U;b40j`LJ!Af;Bnu9BX8Nnke zD#yd7ckP;-Whq<^JSuL3crl(IKYrjOo+l(EK;sZ;-sMR+2`MS5)OIJ57UKE-V7z1N zn+INAu$%%lx^@K34?Axs(%ReG{VT0ee8JmuC7GF-!*_1pyeS}O%@>KickdpN_9-nb zta%(0vR6@5q=>;(Wo2bmS68Q|GD-M&dq0hi?w<25kWV4JHu_7kIE*T;bF6tH@kA>%Ju6pSaB#(rcT=3yLUH-!?*#x zY^nlJAd!|PCO`Z70{)EA*PABC=y5K}0S*p2`CZw@Sp(%hCB?;NxhAHjrfKbuwZt0j z6-Zj$6@K^O76^oA+1WEsj-SDb9~Fwp%g#P>_%NAnFBNwBs^t09W7(%Qz{oE`C* za>i7UXL1rd>U_h8P4CeC$2PH+(ELE>aV9v*pBRoFoF)^s+U=4V%zqF{8C zfSiu5Zb?arE*1+lqT=7RJ3%vj-gxrliPneO5vY?Q0&(={QJe%6H%?v`>6j*5viIHRuHELc4yObg?^Erzc5l8(c(~~mritY`fMwYmx^nh1OO3Hb+>JKbso^u+S(dAl7pLj zf4Noj7%y+MP8wdSsBdm+Y6^jH8LGOdsCZ1G_H%4XO3IfnU+8ptS=j{}o1w);cNAZ5 zPY*RAVSTnoF-d=Na?%F~Wn;rPgW)&bA)RTwwY56)?wu3@F+DvkE-tREt^MfHBbdq> zTUGB_L6{qQZEbDe$9r*fdqAk*;9yunIWadsKelr;Y9K1N$%iCKAi&ahfo~YuswY_}^fV$33kV6^nbZ5f@ z$+iCG-7}m76A4};e(Drd)C>(kxVOSE0DOQREpQ!Bi^M+EPJz>jXa)mN| zgC!3poK0ZE7B(1b|Erlfgg_)WA_wO^`G?I$DbLWoa>C>=P{wT?W zX3!|b1qFm#w>IkpY+!z9&RtKisf&w?BO@aqNls~K1a7a*`B#dNwBTObzEmVgD+!&I zka$~A@T_*ExTFM55`|3#7#N?J2wv-P4JPuU<9Ud*X17 z&`zNLfW4`B*6|DH&fO3vSM<3G#eBTe`~p^lX%*n-*EcY*Dfhm|cy~Qf`JVsMuPaVY z?{N}pL0cqe|eW%6cbOkHxXhJ)9?rQ3f3Jz!_|Yj=D7 zs>E}=)bjeeX3&G#9}8_Zh*=J%F{^$l)f!5=;r3e8@WP1 zo$TxDi$;#JE;RaKQw&+hs=KxB#@g-X4E!Ei;?`e|5$ZV-xj$C&yr z3)(4J)#HncvPJGsK(08tv5AQ{s2;0wZ;-%o^s)Ya%W!RvkuTx*`aWKj;04-xPSH!p zOP!IG6*+nGzAAEeclX4J6Cj#((tvez(+!t=D=tTHN%OH6)O{XV1f&NAm2XoWT>FR< zD}L_$`I{LGs|sI_&a9iKPMrdv5cBi*H_u?)v9}K$H3J5{bcw{60{MhGdzNONJ1{T+ z%2*+>ML$CkemB1>WPQNr!Gj0oJ`2_*uEG>Ot0IROyi~d0^2Cd?7A#DcEG_#3UNMDh z!weW(SX@gX5KzYY`rqN+0CCOTJBxBOs zodPjK495M=mpQQl_`dtb%hvj zqyqI@S64#qh;3o_LiK(ykXcYrkl3c%@*1>NR~LwAe+6oVUEP!Wlixx81Dp6~fF0=V z?ai;+%KGr(2dEhr7qq-QPrteXRYOCABdi9_)8@1SP1o+k(#(n9{`Bb+3~;l(f_qm} zb8}6^i?}$(z(C80JCR7_62$~=ERL+KthBe^kfG>hFw#;}!2^NyILm(xXp4ztW@d&s z?&0md_A}Q4fza?>{I9BN5b)m5uh1+fIe04)*xyB&KkEF23nz{pV{~*ByA4&voi=!g ziP6u1neWUtJr?B0?u7CeqU-QwSTs+fTK*!c>11y4u>boE#T;nPH|^aIm#( z!v0#H+kR*K(vk;YotN7BTpu9B2fI4+3!q9+5FIU0a;QAz@x30?QhM^QO)C7##x|? z`k^){DYwBmuP+Q~2JiUl>K^iC3dM?pD%VZ{(+u702i1Tf2TI-9-G%0bThi8M!UgYc z(SYjB&^Ccf2a#MLx=gY=!IiI5hX%F?YXtd+e z7cO2*(oPv(^5u`B`L9l|H#z$HmVWwVp4|y8ssob)rdvVbheOd_nmHV3(GFy&ib})e zatgu8+uPgCO|JV$$-TaJhM6o(B$MoL?UbdZCGC{9DKC+W{(K08yU3wKzaKw-4D-dy z#=;(+YAkb@Mccrj6=s=*=>QAU)cm|oT04lYxw$#Klo6VLp}X+1IQh4iS{u>P!2vY3 zscDy6)h6qTxr%E)H2UJ&8W@(G++0p>?vlPawWOzd>0F$gaZk8Y27{Rkqn#p; zM(4mCz_hZ4T|)5@m4pNY5`avgI^SDc!ypv|=;Yz*Wlkk7~n zbOETt7#+aCsF)bQ&wF?89*x8TlDOZ$59J3_0!so$glqr9!&cDsvc*9;v9h$(7e)pu zcQIhCqR0&T#9g(&t4n~S_3hiYjg1YE7BK!WeK3Y7ik<@1g=W6Hv-2363NuE=K_7Uj zC0)IGl}Pg^60-^l7MDgFK&b%kJO9k7u`677ko@c!4f(K$7AXD-U2pFmgO2kWLcM@TbypbR_EiEkiKv4GgBZY+6p9A`QK6(NV zsY)r#B?#rc<>dX@xw(VapMjjwPC>ZU*eje9#Dm8($p!$?Nm~LnXz)g(EO7G==s>7? zWktnbU1eiw3HswGAAz&5cmWHY)aj4yShHD zvpcV(RN=dn3Z_C*@(wh&tLr?h0la8sWd)$tL8O5K0Yrz%)fFehBP$ys;1CqF3jqtj zH%bL)3j`Q^6gFuhLjlqU$ygP<>kmv~Xvmw~3Mw1)4qjiMpN&N{WT%jT8uKjEjTNbU zFvA~-wUZZwbU^~rv7Dl!`;eLeqM2l`&h`}XM%shs_;2!QOiZQMlp(lj$Q}X%cUD%4 z_0qF)b3wZU<5+xjA>$+t95_IsP<-b5J?`I+ij4(B>u+ELh5-;tH?2J@JNr^%a(uj! zqGFNfcoXnNxc2R$p47Co0Du1+6K7z%7@ai3Ot3>5BHWQ!Su`5@>h*>D6JYi275o4J z5s1(yf`WpO8tSVh4fghKEsoSiM@NHZ3D+ja$J;9j!Bq0|^QWe!zIyelz|6(pUyY>2 zcI3!OVCkKWLdI0F8JfXh96x^CCU_Hk^3Ja)IY`VjH-?!J2vC-gAI}0xf#_JSaDkL9EtQaztOR-XnZ#V;Iv~y)32z{g zNRX)62A_zj2VLLP1keFPI1ahsqww&`8D}_PL#!zDJ7h-YxyqWFq9iRac>fXJ!|7xb zVCqA_SsG>NLp%#}ps871Q*&uil^7iShHk$HCx7wz%Z4kyNibn3;swX*5K+@ zYEn{5Vbga zzg;2Lq1*fK?fmtt+gn@jji>*NH$&{n$$A;~g+M@4V#Nz{b8~ZYW>;srAOaHO3Ei?Q z>HGiK$5@79WMt%D84U7KSXdZBZck4LgFrukkB^U!hnxnS{)oGEr9X(*@2@W~*F#_m z$tIj)rm6}PNADOKS{NS>ud)4)fQrO=`uH5W_;q%6c5inZ=Fder5!l=?Q!4JRySu!i z;(af#c&zyN*jT9JUBH(87C@^6GWjSI$$fdE1*Au=Ng|cn^YiCV2IGGveM$OYPvUVB zAZ|c{G>ldTZ38bwgooci@ey81@*dt8L;%%^@x8 zsWyy)EQ4@2N#D)c85oK|q=Cwds)Mc$*%=iD4TF;amd@_f6DJRn1q1{j&;ywf6jVt- zDG9|uQU}Zg*a6`f0ma8e`UZN2h2?8Q!?JG$j0Fo*p-Yc~JgvF884LxiBNFSQiiAWK zcyEDXprwWIy$17KXtwib{2G!g0a{p4knP3%=JF)e-NM`)Qk!UOJ3E8R(vUWZlN*UN$Qf)4&En`@6W^$iaU5g&7x+n9MFnhS0)^sH)(_GantsVw zqs;PY?FeMm?>dPvxDX1Ddv0!Qs3z${D3U<8hlA&L$(=n5%g1P^bh~`i1StZR3ib`O zFzjn`^Cl%BVRB(XR7j||tLt34?)uQ^(gtjDa(w)@{7bNN`DTR=LDlH$R{O8Avjp$1b`fd4t*zXz uIqK`{L8ry&Fb^Hraocs Date: Tue, 26 Oct 2021 22:37:55 +0100 Subject: [PATCH 233/331] update --- .gitignore | 4 +++- grid_generated_1.png | Bin 7000 -> 0 bytes 2 files changed, 3 insertions(+), 1 deletion(-) delete mode 100644 grid_generated_1.png diff --git a/.gitignore b/.gitignore index 29d9f6c87b623..eaf67251056f1 100644 --- a/.gitignore +++ b/.gitignore @@ -139,7 +139,9 @@ mnist/ legacy/checkpoints/ *.gz *ubyte -*0.png +grid_generated* +grid_ori* + # pl tests ml-runs/ diff --git a/grid_generated_1.png b/grid_generated_1.png deleted file mode 100644 index 77820f68637fd9ace9e72d03a3af6e1cd155db03..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7000 zcmW+*2{@G9+ne9+ubW){ho`Z~u~k7qAuuTDfm%|F zgOLnH73n(pJt-z8hL8Quz^4c2&!1-vJAgn4a;z;ih-wC`xxhvch}mA+83_sd<`-w- z!%>luhq<}koSZ^yM^usXbA2TaMbw7T@yW?UtgQOiuU|ku4G%vUQG3JO+}zYO0xPa^ zuh(mR-YQAoQ8}?DKi|5tx7C` z60)-t!I)y93v#f==$PKTSyx~0@8@S#6 z&9i6E?rsn5ySln^9XS%PJaOH@VPd*7`^=d$q8I&F*5)jbT$Apq-7d-$J(?L>UU0TS zG^E0Pgo%ae92IX@*xlSb1ABpMUAWWq=3;=x#ft}-YTDWe`ubc*E>jr_0>Q$<@)#@5 z9R2##I`tG7UPlNrM`Hv_N(qlu8F#W z!tAOVYbWtlt zJzy=Tq{PSHf9t8R`g>!dx3_Y*_R`v#G8M0v?s@m_;fPvENy)PE@+hqMv)3~Kw(U;b40j`LJ!Af;Bnu9BX8Nnke zD#yd7ckP;-Whq<^JSuL3crl(IKYrjOo+l(EK;sZ;-sMR+2`MS5)OIJ57UKE-V7z1N zn+INAu$%%lx^@K34?Axs(%ReG{VT0ee8JmuC7GF-!*_1pyeS}O%@>KickdpN_9-nb zta%(0vR6@5q=>;(Wo2bmS68Q|GD-M&dq0hi?w<25kWV4JHu_7kIE*T;bF6tH@kA>%Ju6pSaB#(rcT=3yLUH-!?*#x zY^nlJAd!|PCO`Z70{)EA*PABC=y5K}0S*p2`CZw@Sp(%hCB?;NxhAHjrfKbuwZt0j z6-Zj$6@K^O76^oA+1WEsj-SDb9~Fwp%g#P>_%NAnFBNwBs^t09W7(%Qz{oE`C* za>i7UXL1rd>U_h8P4CeC$2PH+(ELE>aV9v*pBRoFoF)^s+U=4V%zqF{8C zfSiu5Zb?arE*1+lqT=7RJ3%vj-gxrliPneO5vY?Q0&(={QJe%6H%?v`>6j*5viIHRuHELc4yObg?^Erzc5l8(c(~~mritY`fMwYmx^nh1OO3Hb+>JKbso^u+S(dAl7pLj zf4Noj7%y+MP8wdSsBdm+Y6^jH8LGOdsCZ1G_H%4XO3IfnU+8ptS=j{}o1w);cNAZ5 zPY*RAVSTnoF-d=Na?%F~Wn;rPgW)&bA)RTwwY56)?wu3@F+DvkE-tREt^MfHBbdq> zTUGB_L6{qQZEbDe$9r*fdqAk*;9yunIWadsKelr;Y9K1N$%iCKAi&ahfo~YuswY_}^fV$33kV6^nbZ5f@ z$+iCG-7}m76A4};e(Drd)C>(kxVOSE0DOQREpQ!Bi^M+EPJz>jXa)mN| zgC!3poK0ZE7B(1b|Erlfgg_)WA_wO^`G?I$DbLWoa>C>=P{wT?W zX3!|b1qFm#w>IkpY+!z9&RtKisf&w?BO@aqNls~K1a7a*`B#dNwBTObzEmVgD+!&I zka$~A@T_*ExTFM55`|3#7#N?J2wv-P4JPuU<9Ud*X17 z&`zNLfW4`B*6|DH&fO3vSM<3G#eBTe`~p^lX%*n-*EcY*Dfhm|cy~Qf`JVsMuPaVY z?{N}pL0cqe|eW%6cbOkHxXhJ)9?rQ3f3Jz!_|Yj=D7 zs>E}=)bjeeX3&G#9}8_Zh*=J%F{^$l)f!5=;r3e8@WP1 zo$TxDi$;#JE;RaKQw&+hs=KxB#@g-X4E!Ei;?`e|5$ZV-xj$C&yr z3)(4J)#HncvPJGsK(08tv5AQ{s2;0wZ;-%o^s)Ya%W!RvkuTx*`aWKj;04-xPSH!p zOP!IG6*+nGzAAEeclX4J6Cj#((tvez(+!t=D=tTHN%OH6)O{XV1f&NAm2XoWT>FR< zD}L_$`I{LGs|sI_&a9iKPMrdv5cBi*H_u?)v9}K$H3J5{bcw{60{MhGdzNONJ1{T+ z%2*+>ML$CkemB1>WPQNr!Gj0oJ`2_*uEG>Ot0IROyi~d0^2Cd?7A#DcEG_#3UNMDh z!weW(SX@gX5KzYY`rqN+0CCOTJBxBOs zodPjK495M=mpQQl_`dtb%hvj zqyqI@S64#qh;3o_LiK(ykXcYrkl3c%@*1>NR~LwAe+6oVUEP!Wlixx81Dp6~fF0=V z?ai;+%KGr(2dEhr7qq-QPrteXRYOCABdi9_)8@1SP1o+k(#(n9{`Bb+3~;l(f_qm} zb8}6^i?}$(z(C80JCR7_62$~=ERL+KthBe^kfG>hFw#;}!2^NyILm(xXp4ztW@d&s z?&0md_A}Q4fza?>{I9BN5b)m5uh1+fIe04)*xyB&KkEF23nz{pV{~*ByA4&voi=!g ziP6u1neWUtJr?B0?u7CeqU-QwSTs+fTK*!c>11y4u>boE#T;nPH|^aIm#( z!v0#H+kR*K(vk;YotN7BTpu9B2fI4+3!q9+5FIU0a;QAz@x30?QhM^QO)C7##x|? z`k^){DYwBmuP+Q~2JiUl>K^iC3dM?pD%VZ{(+u702i1Tf2TI-9-G%0bThi8M!UgYc z(SYjB&^Ccf2a#MLx=gY=!IiI5hX%F?YXtd+e z7cO2*(oPv(^5u`B`L9l|H#z$HmVWwVp4|y8ssob)rdvVbheOd_nmHV3(GFy&ib})e zatgu8+uPgCO|JV$$-TaJhM6o(B$MoL?UbdZCGC{9DKC+W{(K08yU3wKzaKw-4D-dy z#=;(+YAkb@Mccrj6=s=*=>QAU)cm|oT04lYxw$#Klo6VLp}X+1IQh4iS{u>P!2vY3 zscDy6)h6qTxr%E)H2UJ&8W@(G++0p>?vlPawWOzd>0F$gaZk8Y27{Rkqn#p; zM(4mCz_hZ4T|)5@m4pNY5`avgI^SDc!ypv|=;Yz*Wlkk7~n zbOETt7#+aCsF)bQ&wF?89*x8TlDOZ$59J3_0!so$glqr9!&cDsvc*9;v9h$(7e)pu zcQIhCqR0&T#9g(&t4n~S_3hiYjg1YE7BK!WeK3Y7ik<@1g=W6Hv-2363NuE=K_7Uj zC0)IGl}Pg^60-^l7MDgFK&b%kJO9k7u`677ko@c!4f(K$7AXD-U2pFmgO2kWLcM@TbypbR_EiEkiKv4GgBZY+6p9A`QK6(NV zsY)r#B?#rc<>dX@xw(VapMjjwPC>ZU*eje9#Dm8($p!$?Nm~LnXz)g(EO7G==s>7? zWktnbU1eiw3HswGAAz&5cmWHY)aj4yShHD zvpcV(RN=dn3Z_C*@(wh&tLr?h0la8sWd)$tL8O5K0Yrz%)fFehBP$ys;1CqF3jqtj zH%bL)3j`Q^6gFuhLjlqU$ygP<>kmv~Xvmw~3Mw1)4qjiMpN&N{WT%jT8uKjEjTNbU zFvA~-wUZZwbU^~rv7Dl!`;eLeqM2l`&h`}XM%shs_;2!QOiZQMlp(lj$Q}X%cUD%4 z_0qF)b3wZU<5+xjA>$+t95_IsP<-b5J?`I+ij4(B>u+ELh5-;tH?2J@JNr^%a(uj! zqGFNfcoXnNxc2R$p47Co0Du1+6K7z%7@ai3Ot3>5BHWQ!Su`5@>h*>D6JYi275o4J z5s1(yf`WpO8tSVh4fghKEsoSiM@NHZ3D+ja$J;9j!Bq0|^QWe!zIyelz|6(pUyY>2 zcI3!OVCkKWLdI0F8JfXh96x^CCU_Hk^3Ja)IY`VjH-?!J2vC-gAI}0xf#_JSaDkL9EtQaztOR-XnZ#V;Iv~y)32z{g zNRX)62A_zj2VLLP1keFPI1ahsqww&`8D}_PL#!zDJ7h-YxyqWFq9iRac>fXJ!|7xb zVCqA_SsG>NLp%#}ps871Q*&uil^7iShHk$HCx7wz%Z4kyNibn3;swX*5K+@ zYEn{5Vbga zzg;2Lq1*fK?fmtt+gn@jji>*NH$&{n$$A;~g+M@4V#Nz{b8~ZYW>;srAOaHO3Ei?Q z>HGiK$5@79WMt%D84U7KSXdZBZck4LgFrukkB^U!hnxnS{)oGEr9X(*@2@W~*F#_m z$tIj)rm6}PNADOKS{NS>ud)4)fQrO=`uH5W_;q%6c5inZ=Fder5!l=?Q!4JRySu!i z;(af#c&zyN*jT9JUBH(87C@^6GWjSI$$fdE1*Au=Ng|cn^YiCV2IGGveM$OYPvUVB zAZ|c{G>ldTZ38bwgooci@ey81@*dt8L;%%^@x8 zsWyy)EQ4@2N#D)c85oK|q=Cwds)Mc$*%=iD4TF;amd@_f6DJRn1q1{j&;ywf6jVt- zDG9|uQU}Zg*a6`f0ma8e`UZN2h2?8Q!?JG$j0Fo*p-Yc~JgvF884LxiBNFSQiiAWK zcyEDXprwWIy$17KXtwib{2G!g0a{p4knP3%=JF)e-NM`)Qk!UOJ3E8R(vUWZlN*UN$Qf)4&En`@6W^$iaU5g&7x+n9MFnhS0)^sH)(_GantsVw zqs;PY?FeMm?>dPvxDX1Ddv0!Qs3z${D3U<8hlA&L$(=n5%g1P^bh~`i1StZR3ib`O zFzjn`^Cl%BVRB(XR7j||tLt34?)uQ^(gtjDa(w)@{7bNN`DTR=LDlH$R{O8Avjp$1b`fd4t*zXz uIqK`{L8ry&Fb^Hraocs Date: Tue, 26 Oct 2021 21:35:30 +0100 Subject: [PATCH 234/331] Revert "switch to Any" This reverts commit fd8660c10102dada28d998736173bc34df892916. --- pytorch_lightning/lite/lite.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 18f069bf945e1..170cf226b3b5a 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -16,7 +16,7 @@ from contextlib import contextmanager from functools import partial from pathlib import Path -from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, Sequence, Tuple, Union +from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, overload, Sequence, Tuple, Union import torch import torch.nn as nn @@ -275,7 +275,19 @@ def cast(self) -> Generator[None, None, None]: with self._precision_plugin.forward_context(): yield + @overload + def to_device(self, obj: nn.Module) -> nn.Module: + ... + + @overload + def to_device(self, obj: Tensor) -> Tensor: + ... + + @overload def to_device(self, obj: Any) -> Any: + ... + + def to_device(self, obj: Union[nn.Module, Tensor, Any]) -> Union[nn.Module, Tensor, Any]: """Move a :class:`torch.nn.Module` or a collection of tensors to the current device, if it is not already on that device. From 2ccae2791956037c44b323ceddec7a6126d1236f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 27 Oct 2021 01:26:30 +0200 Subject: [PATCH 235/331] try to fix mypy --- pytorch_lightning/lite/lite.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 170cf226b3b5a..a1291ce57ab87 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -283,10 +283,6 @@ def to_device(self, obj: nn.Module) -> nn.Module: def to_device(self, obj: Tensor) -> Tensor: ... - @overload - def to_device(self, obj: Any) -> Any: - ... - def to_device(self, obj: Union[nn.Module, Tensor, Any]) -> Union[nn.Module, Tensor, Any]: """Move a :class:`torch.nn.Module` or a collection of tensors to the current device, if it is not already on that device. From 0f11f708b2f685c68a96911e01dafa40c3ef0b84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 27 Oct 2021 01:28:26 +0200 Subject: [PATCH 236/331] x --- pytorch_lightning/lite/lite.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index a1291ce57ab87..49798b138567e 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -275,14 +275,6 @@ def cast(self) -> Generator[None, None, None]: with self._precision_plugin.forward_context(): yield - @overload - def to_device(self, obj: nn.Module) -> nn.Module: - ... - - @overload - def to_device(self, obj: Tensor) -> Tensor: - ... - def to_device(self, obj: Union[nn.Module, Tensor, Any]) -> Union[nn.Module, Tensor, Any]: """Move a :class:`torch.nn.Module` or a collection of tensors to the current device, if it is not already on that device. From 782c70f46996af99b71679b72ad7bb00254e23f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 27 Oct 2021 09:25:36 +0200 Subject: [PATCH 237/331] lightning lite package and tests --- pytorch_lightning/lite/__init__.py | 17 ++ pytorch_lightning/lite/lite.py | 470 +++++++++++++++++++++++++++++ pytorch_lightning/lite/wrappers.py | 151 +++++++++ tests/lite/__init__.py | 0 tests/lite/test_lite.py | 392 ++++++++++++++++++++++++ tests/lite/test_parity.py | 237 +++++++++++++++ tests/lite/test_wrappers.py | 106 +++++++ 7 files changed, 1373 insertions(+) create mode 100644 pytorch_lightning/lite/__init__.py create mode 100644 pytorch_lightning/lite/lite.py create mode 100644 pytorch_lightning/lite/wrappers.py create mode 100644 tests/lite/__init__.py create mode 100644 tests/lite/test_lite.py create mode 100644 tests/lite/test_parity.py create mode 100644 tests/lite/test_wrappers.py diff --git a/pytorch_lightning/lite/__init__.py b/pytorch_lightning/lite/__init__.py new file mode 100644 index 0000000000000..f4634fe54e548 --- /dev/null +++ b/pytorch_lightning/lite/__init__.py @@ -0,0 +1,17 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pytorch_lightning.lite.lite import LightningLite + +__all__ = ["LightningLite"] diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py new file mode 100644 index 0000000000000..49798b138567e --- /dev/null +++ b/pytorch_lightning/lite/lite.py @@ -0,0 +1,470 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from abc import ABC, abstractmethod +from contextlib import contextmanager +from functools import partial +from pathlib import Path +from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, overload, Sequence, Tuple, Union + +import torch +import torch.nn as nn +from torch import Tensor +from torch.optim import Optimizer +from torch.utils.data import DataLoader, DistributedSampler, RandomSampler, SequentialSampler + +from pytorch_lightning import Trainer +from pytorch_lightning.accelerators import Accelerator +from pytorch_lightning.lite.wrappers import _LiteDataLoader, _LiteModule, _LiteOptimizer +from pytorch_lightning.plugins import ( + DDPShardedPlugin, + DDPSpawnPlugin, + DeepSpeedPlugin, + PLUGIN_INPUT, + TPUSpawnPlugin, + TrainingTypePlugin, +) +from pytorch_lightning.trainer.connectors.accelerator_connector import AcceleratorConnector +from pytorch_lightning.trainer.data_loading import TrainerDataLoadingMixin +from pytorch_lightning.utilities import DeviceType, DistributedType, move_data_to_device +from pytorch_lightning.utilities.apply_func import apply_to_collection, convert_to_tensors +from pytorch_lightning.utilities.data import has_iterable_dataset +from pytorch_lightning.utilities.exceptions import MisconfigurationException + + +class LightningLite(ABC): + """Lite accelerates your PyTorch training or inference code with minimal changes required. + + - Automatic placement of models and data onto the device + - Automatic support for mixed and double precision (smaller memory footprint) + - Seamless switching between hardware (CPU, GPU, TPU) and distributed training strategies + (data-parallel training, sharded training, etc.) + - Automated spawning of processes, no launch utilities required + - Multi-node support + + Args: + accelerator: The hardware to run on. Possible choices are: cpu, gpu, tpu, auto. + strategy: Strategy for how to run across multiple devices. Possible choices are: + dp, ddp, ddp_spawn, tpu_spawn, deepspeed, ddp_sharded. + devices: Number of devices to train on (int) or which GPUs to train on (list or str). The value applies + per node. + num_nodes: Number of GPU nodes for distributed training. + precision: Double precision (64), full precision (32), half precision (16) or bfloat16 precision (bf16). + plugins: One or several custom plugins + gpus: Provides the same function as the ``devices`` argument but implies ``accelerator="gpu"``. + tpu_cores: Provides the same function as the ``devices`` argument but implies ``accelerator="tpu"``. + """ + + def __init__( + self, + accelerator: Optional[Union[str, Accelerator]] = None, + strategy: Optional[Union[str, TrainingTypePlugin]] = None, + devices: Optional[Union[List[int], str, int]] = None, + num_nodes: int = 1, + precision: Union[int, str] = 32, + plugins: Optional[Union[PLUGIN_INPUT, List[PLUGIN_INPUT]]] = None, + gpus: Optional[Union[List[int], str, int]] = None, + tpu_cores: Optional[Union[List[int], str, int]] = None, + ) -> None: + self._check_accelerator_support(accelerator) + self._check_strategy_support(strategy) + gpu_ids, tpu_cores = Trainer._parse_devices(gpus=gpus, auto_select_gpus=False, tpu_cores=tpu_cores) + self._accelerator_connector = AcceleratorConnector( + num_processes=1, + devices=devices, + tpu_cores=tpu_cores, + ipus=None, + accelerator=accelerator, + strategy=strategy, + gpus=gpus, + gpu_ids=gpu_ids, + num_nodes=num_nodes, + sync_batchnorm=False, # TODO: add support? + benchmark=False, + replace_sampler_ddp=True, + deterministic=False, + precision=precision, + amp_type="native", + amp_level=None, + plugins=plugins, + ) + self._accelerator = self._accelerator_connector.accelerator + self._strategy = self._accelerator.training_type_plugin + self._precision_plugin = self._accelerator.precision_plugin + self._num_models: int = 0 + + # wrap the run method so we can inject setup logic or spawn processes for the user + setattr(self, "run", self._run_wrapper(self.run)) + + @property + def device(self) -> torch.device: + """The current device this process runs on. + + Use this to create tensors directly on the device if needed. + """ + return self._accelerator.root_device + + @property + def global_rank(self) -> int: + """The global index of the current process across all devices and nodes.""" + return getattr(self._strategy, "global_rank", 0) + + @property + def local_rank(self) -> int: + """The index of the current process among the processes running on the local node.""" + return getattr(self._strategy, "local_rank", 0) + + @property + def node_rank(self) -> int: + """The index of the current node.""" + return getattr(self._strategy, "node_rank", 0) + + @property + def world_size(self) -> int: + """The total number of processes running across all devices and nodes.""" + return getattr(self._strategy, "world_size", 1) + + @property + def is_global_zero(self) -> bool: + """Wether this rank is rank zero.""" + return self._strategy.is_global_zero + + @abstractmethod + def run(self, *args: Any, **kwargs: Any) -> Any: + """All the code inside this run method gets accelerated by Lite. + + Args: + *args: Add any positional arguments you need, e.g., the hyperparameters for your model + **kwargs: Add any keyword arguments you need, e.g., the hyperparameters for your model + """ + + def setup( + self, + model: nn.Module, + *optimizers: Optimizer, + move_to_device: bool = True, + ) -> Union[_LiteModule, List[Union[_LiteModule, _LiteOptimizer]]]: + """Setup a model and its optimizers for accelerated training. + + Args: + model: A model to setup + *optimizers: The optimizer(s) to setup (no optimizers is also possible) + move_to_device: If set ``True`` (default), moves the model to the correct device. Set this to ``False`` + and alternatively use :meth:`to_device` manually. + + Returns: + The tuple of the wrapped model and list of optimizers, in the same order they were passed in. + """ + self._validate_setup(model, optimizers) + + if move_to_device: + model = self._move_model_to_device(model=model, optimizers=list(optimizers)) + + # Let accelerator/plugin wrap and connect the models and optimizers + model, optimizers = self._strategy._setup_model_and_optimizers(model, list(optimizers)) + model = _LiteModule(model, self._accelerator) + optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self._accelerator) for optimizer in optimizers] + self._num_models += 1 + if optimizers: + return [model] + optimizers # type: ignore + return model + + def setup_dataloaders( + self, *dataloaders: DataLoader, replace_sampler: bool = True, move_to_device: bool = True + ) -> Union[DataLoader, List[DataLoader], Iterable]: + """Setup one or multiple dataloaders for accelerated training. If you need different settings for each + dataloader, call this method individually for each one. + + Args: + *dataloaders: A single dataloader or a sequence of dataloaders. + replace_sampler: If set ``True`` (default), automatically wraps or replaces the sampler on the dataloader(s) + for distributed training. If you have a custom sampler defined, set this to this argument to ``False``. + move_to_device: If set ``True`` (default), moves the data returned by the dataloader(s) automatially to + the correct device. Set this to ``False`` and alternatively use :meth:`to_device` manually on the + returned data. + + Returns: + The wrapped dataloaders, in the same order they were passed in. + """ + self._validate_setup_dataloaders(dataloaders) + dataloaders = [ + self._setup_dataloader(dataloader, replace_sampler=replace_sampler, move_to_device=move_to_device) + for dataloader in dataloaders + ] + dataloaders = dataloaders[0] if len(dataloaders) == 1 else dataloaders + return dataloaders + + def _setup_dataloader( + self, dataloader: DataLoader, replace_sampler: bool = True, move_to_device: bool = True + ) -> Union[Iterable, DataLoader]: + """Setup a single dataloader for accelerated training. + + Args: + dataloader: The dataloader to accelerate. + replace_sampler: If set ``True`` (default), automatically wraps or replaces the sampler on the dataloader + for distributed training. If you have a custom sampler defined, set this to this argument to ``False``. + move_to_device: If set ``True`` (default), moves the data returned by the dataloader automatially to + the correct device. Set this to ``False`` and alternatively use :meth:`to_device` manually on the + returned data. + + Returns: + The wrapped dataloader. + """ + sampler = dataloader.sampler + if replace_sampler and self._requires_distributed_sampler(dataloader): + if not isinstance(sampler, (SequentialSampler, RandomSampler)): + raise MisconfigurationException( + "You seem to have configured a sampler in your DataLoader. This will be replaced " + " by `DistributedSampler` since `replace_sampler_ddp` is True and you are using" + " distributed training. Either remove the sampler from your DataLoader or set" + " `replace_sampler=False` if you want to use your custom sampler." + ) + sampler = self._get_distributed_sampler(dataloader, **self._strategy.distributed_sampler_kwargs) + + kwargs = TrainerDataLoadingMixin._get_dataloader_init_kwargs(dataloader, sampler) + device = self.device if move_to_device else None + if isinstance(self._strategy, TPUSpawnPlugin): + dataloader = DataLoader(**kwargs) + else: + dataloader = _LiteDataLoader(device=device, **kwargs) + return self._strategy.process_dataloader(dataloader) + + def backward(self, tensor: Tensor, *args: Any, model: Optional[_LiteModule] = None, **kwargs: Any) -> None: + """Replaces ``loss.backward()`` in your training loop. Handles precision and automatically for you. + + Args: + tensor: The tensor (loss) to back-propagate gradients from. + *args: Optional positional arguments passed to the underlying backward function. + model: Optional model instance for plugins that require the model for backward(). + **kwargs: Optional named keyword arguments passed to the underlying backward function. + + Note: + When using ``strategy='deepspeed'`` and multiple models were setup, it is required to pass in the + model as argument here. + """ + module = model.module if model is not None else model + if self._num_models > 0 and isinstance(self._strategy, DeepSpeedPlugin): + if model is None: + raise MisconfigurationException( + "When using multiple models + deepspeed, please provide the model used to perform the optimization." + ) + + # requires to attach the current `DeepSpeedEngine` for the `_LiteOptimizer.step` call. + self._strategy.model = module + + self._precision_plugin._run_backward(tensor, module, *args, **kwargs) + + @contextmanager + def cast(self) -> Generator[None, None, None]: + """A context manager to automatically convert operations for the chosen precision. + + Use this only if the `forward` method of your model does not cover all operations you wish to run with the + chosen precision setting. + """ + with self._precision_plugin.forward_context(): + yield + + def to_device(self, obj: Union[nn.Module, Tensor, Any]) -> Union[nn.Module, Tensor, Any]: + """Move a :class:`torch.nn.Module` or a collection of tensors to the current device, if it is not already + on that device. + + Args: + obj: An object to move to the device. Can be an instance of :class:`torch.nn.Module`, a tensor, or a + (nested) collection of tensors (e.g., a dictionary). + + Returns: + A reference to the object that was moved to the new device. + """ + if isinstance(obj, nn.Module): + if self.device.type == "cuda": + # need to call this manually here again in case we spawned with DDPSpawnPlugin + # TODO: refactor to let plugin handle this cleanly + torch.cuda.set_device(self.device) + return obj.to(self.device) + return move_data_to_device(obj, device=self.device) + + def print(self, *args: Any, **kwargs: Any) -> None: + """Print something only on the first process. + + Arguments passed to this method are forwarded to the Python built-in :func:`print` function. + """ + if self.local_rank == 0: + print(*args, **kwargs) + + def barrier(self, name: Optional[str] = None) -> None: + """Wait for all processes to enter this call. Use this to synchronize all parallel processes, but only if + necessary, otherwise the overhead of synchronization will cause your program to slow down. + + Example:: + + if self.global_rank == 0: + # let process 0 download the dataset + dataset.download_files() + + # let all processes wait before reading the dataset + self.barrier() + + # now all processes can read the files and start training + """ + self._strategy.barrier() + + def all_gather( + self, data: Union[torch.Tensor, Dict, List, Tuple], group: Optional[Any] = None, sync_grads: bool = False + ) -> Union[torch.Tensor, Dict, List, Tuple]: + r""" + Gather tensors or collections of tensors from multiple processes. + + Args: + data: int, float, tensor of shape (batch, ...), or a (possibly nested) collection thereof. + group: the process group to gather results from. Defaults to all processes (world) + sync_grads: flag that allows users to synchronize gradients for the all_gather operation + + Return: + A tensor of shape (world_size, batch, ...), or if the input was a collection + the output will also be a collection with tensors of this shape. + """ + group = group if group is not None else torch.distributed.group.WORLD + data = convert_to_tensors(data, device=self.device) + return apply_to_collection(data, torch.Tensor, self._strategy.all_gather, group=group, sync_grads=sync_grads) + + def broadcast(self, obj: object, src: int = 0) -> object: + return self._strategy.broadcast(obj, src=src) + + def save_checkpoint(self, filepath: Union[str, Path], content: Dict[str, Any]) -> None: + """Save a checkpoint contents to a file. + + How and which processes save gets determined by the `strategy`. For example, the `ddp` strategy + saves checkpoints only on process 0. + + Args: + filepath: A path to where the file should be saved + content: A dictionary with contents, i.e., the state dict of your model + """ + self._strategy.save_checkpoint(content, filepath) + + def _run_wrapper(self, run_method: Callable) -> Callable: + return partial(self._run_impl, run_method) + + def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> Any: + self._set_plugin_specific_precision_variables() + self._accelerator.setup_environment() + + # apply sharded context to prevent OOM + run_method = partial(self._run_with_sharded_context, run_method) + + if isinstance(self._strategy, DDPSpawnPlugin): + return self._strategy.spawn(run_method, *args, **kwargs) + else: + return run_method(*args, **kwargs) + + def _run_with_sharded_context(self, run_method: Callable, *args: Any, **kwargs: Any) -> Any: + with self._strategy.model_sharded_context(): + return run_method(*args, **kwargs) + + def _set_plugin_specific_precision_variables(self) -> None: + # todo: these are hacks as plugins rely on access to the precision plugin + if isinstance(self._strategy, DeepSpeedPlugin): + self._set_deepspeed_precision_variables() + if isinstance(self._strategy, DDPShardedPlugin): + self._strategy._precision = self._accelerator_connector.precision + + def _move_model_to_device(self, model: nn.Module, optimizers: List[Optimizer]) -> nn.Module: + if isinstance(self._strategy, TPUSpawnPlugin): + # When the user creates the optimizer, they reference the parameters on the CPU. + # However, when running with TPU the parameters get copied and the reference in the optimizer + # remains invalid. We need to update the references to point to the parameter tensors on the device. + params_on_cpu = dict(model.named_parameters()) + model = self.to_device(model) + params_on_device = dict(model.named_parameters()) + + mapping = {param: params_on_device[name] for name, param in params_on_cpu.items()} + for optimizer in optimizers: + for param_group in optimizer.param_groups: + param_group["params"] = [mapping.get(p, p) for p in param_group["params"]] + else: + model = self.to_device(model) + return model + + def _set_deepspeed_precision_variables(self) -> None: + # TODO: Refactor this once precision pluging is part of the strategy. + amp_type = self._accelerator_connector.amp_type + amp_level = self._accelerator_connector.amp_level + precision = self._accelerator_connector.precision + self._strategy.amp_level, self._strategy.amp_type, self._strategy._precision = amp_level, amp_type, precision + + def _requires_distributed_sampler(self, dataloader: DataLoader) -> bool: + return ( + self._accelerator_connector.is_distributed + and not isinstance(dataloader.sampler, DistributedSampler) + and not has_iterable_dataset(dataloader) + ) + + @staticmethod + def _get_distributed_sampler(dataloader: DataLoader, **kwargs: Any) -> DistributedSampler: + kwargs.setdefault("seed", int(os.getenv("PL_GLOBAL_SEED", 0))) + return DistributedSampler(dataloader.dataset, **kwargs) + + def _check_accelerator_support(self, accelerator: Optional[Union[str, Accelerator]]) -> None: + supported = [t.value.lower() for t in self._supported_device_types()] + ["auto"] + valid = accelerator is None or isinstance(accelerator, Accelerator) or accelerator in supported + if not valid: + raise MisconfigurationException( + f"`accelerator={repr(accelerator)}` is not a valid choice." + f" Choose one of {supported} or pass in a `Accelerator` instance." + ) + + def _check_strategy_support(self, strategy: Optional[Union[str, TrainingTypePlugin]]) -> None: + supported = [t.lower() for t in self._supported_strategy_types()] + valid = strategy is None or isinstance(strategy, TrainingTypePlugin) or strategy in supported + if not valid: + raise MisconfigurationException( + f"`strategy={repr(strategy)}` is not a valid choice." + f" Choose one of {supported} or pass in a `TrainingTypePlugin` instance." + ) + + @staticmethod + def _supported_device_types() -> Sequence[DeviceType]: + return ( + DeviceType.CPU, + DeviceType.GPU, + DeviceType.TPU, + ) + + @staticmethod + def _supported_strategy_types() -> Sequence[str]: + return ( + DistributedType.DP, + DistributedType.DDP, + DistributedType.DDP_SPAWN, + DistributedType.TPU_SPAWN, + DistributedType.DEEPSPEED, + DistributedType.DDP_SHARDED, + DistributedType.DDP_SHARDED_SPAWN, + ) + + @staticmethod + def _validate_setup(model: nn.Module, optimizers: Sequence[Optimizer]) -> None: + if isinstance(model, _LiteModule): + raise MisconfigurationException("A model should be passed only once to the `setup` method.") + + if any(isinstance(opt, _LiteOptimizer) for opt in optimizers): + raise MisconfigurationException("An optimizer should be passed only once to the `setup` method.") + + @staticmethod + def _validate_setup_dataloaders(dataloaders: Sequence[DataLoader]) -> None: + if any(isinstance(dl, _LiteDataLoader) for dl in dataloaders): + raise MisconfigurationException("A dataloader should be passed only once to the `setup_dataloaders` method") + + if any(not isinstance(dl, DataLoader) for dl in dataloaders): + raise MisconfigurationException("Only PyTorch DataLoader are currently supported in `setup_dataloaders`.") diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py new file mode 100644 index 0000000000000..e1d16ca8a3384 --- /dev/null +++ b/pytorch_lightning/lite/wrappers.py @@ -0,0 +1,151 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any, Callable, Dict, Generator, Iterator, List, Optional, Union + +import torch +from torch import nn as nn +from torch import Tensor +from torch.optim import Optimizer +from torch.utils.data import DataLoader + +from pytorch_lightning.accelerators import Accelerator +from pytorch_lightning.utilities.apply_func import apply_to_collection, move_data_to_device + + +def _do_nothing_closure() -> None: + return None + + +class _LiteOptimizer: + def __init__(self, optimizer: Optimizer, accelerator: Accelerator) -> None: + """LiteOptimizer is a thin wrapper around the :class:`~torch.optim.Optimizer` that delegates the optimizer + step calls to the accelerator/strategy plugin. + + The underlying wrapped optimizer object can be accessed via the property :attr:`optimizer`. + + Args: + optimizer: The optimizer to wrap + accelerator: Reference to the accelerator for handling the optimizer step + """ + self.__dict__ = {k: v for k, v in optimizer.__dict__.items() if k not in ("step", "__del__")} + self.__class__ = type("Lite" + optimizer.__class__.__name__, (self.__class__, optimizer.__class__), {}) + self._optimizer = optimizer + self._accelerator = accelerator + + @property + def optimizer(self) -> Optimizer: + return self._optimizer + + @property + def state(self) -> Dict[str, torch.Tensor]: + return self._optimizer.state + + @state.setter + def state(self, state: Dict[str, torch.Tensor]) -> None: + self._optimizer.state = state + + @property + def defaults(self) -> Dict[str, Any]: + return self._optimizer.defaults + + @defaults.setter + def defaults(self, defaults: Dict[str, Any]) -> None: + self._optimizer.defaults = defaults + + @property + def param_groups(self) -> List[Dict[str, torch.Tensor]]: + return self._optimizer.param_groups + + @param_groups.setter + def param_groups(self, param_groups: List[Dict[str, torch.Tensor]]) -> None: + self._optimizer.param_groups = param_groups + + def step(self, closure: Optional[Callable] = None) -> None: + closure = closure or _do_nothing_closure + self._accelerator.optimizer_step( + self._optimizer, + opt_idx=0, + lambda_closure=closure, + model=self._accelerator.model, + ) + + def zero_grad(self, *args: Any, **kwargs: Any) -> None: + self._optimizer.zero_grad(*args, **kwargs) + + +class _LiteModule(nn.Module): + # TODO: Pass in the precision plugin instead of accelerator + def __init__(self, module: nn.Module, accelerator: Accelerator) -> None: + """The LiteModule is a thin wrapper around the :class:`torch.nn.Module` and handles precision / autocast + automatically for the forward pass. + + The underlying wrapped module can be accessed via the property :attr:`module`. + + Args: + module: The module to wrap + accelerator: Reference to the accelerator for handling precision context + """ + super().__init__() + self._module = module + self._accelerator = accelerator + + @property + def module(self) -> nn.Module: + return self._module + + def forward(self, *args: Any, **kwargs: Any) -> Any: + """Casts all inputs to the right precision and handles autocast for operations in the module forward + method.""" + precision = self._accelerator.precision_plugin.precision + precision_to_type = { + "mixed": torch.float16, + 16: torch.float16, + 32: torch.float32, + 64: torch.float64, + } + # TODO (@awaelchli): let the precision plugin handle the conversion + to_type = precision_to_type[precision] + args, kwargs = apply_to_collection([args, kwargs], function=lambda t: t.to(to_type), dtype=Tensor) + + with self._accelerator.precision_plugin.forward_context(): + output = self.module(*args, **kwargs) + + output = apply_to_collection(output, function=lambda t: t.to(torch.get_default_dtype()), dtype=Tensor) + return output + + +class _LiteDataLoader(DataLoader): + def __init__(self, device: Optional[torch.device] = None, **dl_kwargs: Any) -> None: + """The LiteDataLoader is an extension of the PyTorch :class:`~torch.utils.data.DataLoader` that adds + additional features such as moving the data to the device automatically. + + Args: + device: The device to which the data should be moved. By default the device is `None` and no data + transfers will be made (identical behavior as :class:`~torch.utils.data.DataLoader`). + **dl_kwargs: Accepts all arguments that the PyTorch :class:`~torch.utils.data.DataLoader` accepts. + """ + super().__init__(**dl_kwargs) + self._device = device + + @property + def device(self) -> Optional[torch.device]: + return self._device + + def __iter__(self) -> Union[Iterator[Any], Generator[Any, None, None]]: + iterator = super().__iter__() + if self._device is None: + return iterator + + for item in iterator: + yield move_data_to_device(item, self._device) diff --git a/tests/lite/__init__.py b/tests/lite/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py new file mode 100644 index 0000000000000..def9ce29ac9dc --- /dev/null +++ b/tests/lite/test_lite.py @@ -0,0 +1,392 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from copy import deepcopy +from unittest import mock +from unittest.mock import Mock, PropertyMock + +import pytest +import torch +import torch.distributed +import torch.nn.functional +from torch import nn +from torch.utils.data import DataLoader, DistributedSampler, Sampler + +from pytorch_lightning import seed_everything +from pytorch_lightning.lite import LightningLite +from pytorch_lightning.lite.wrappers import _LiteDataLoader, _LiteModule, _LiteOptimizer +from pytorch_lightning.plugins import DeepSpeedPlugin, PrecisionPlugin, TrainingTypePlugin +from pytorch_lightning.utilities import DistributedType +from pytorch_lightning.utilities.exceptions import MisconfigurationException +from tests.helpers.runif import RunIf + + +class EmptyLite(LightningLite): + def run(self): + pass + + +class BoringModel(nn.Module): + def __init__(self): + super().__init__() + self.layer = torch.nn.Linear(32, 2, bias=False) + + def forward(self, x): + x = self.layer(x) + return torch.nn.functional.mse_loss(x, torch.ones_like(x)) + + +@pytest.mark.parametrize("accelerator", ["coconut"]) +def test_unsupported_accelerator(accelerator): + with pytest.raises(MisconfigurationException, match=f"`accelerator={repr(accelerator)}` is not a valid choice"): + EmptyLite(accelerator=accelerator) + + +@pytest.mark.parametrize("strategy", ["coconut"]) +def test_unsupported_strategy(strategy): + with pytest.raises(MisconfigurationException, match=f"`strategy={repr(strategy)}` is not a valid choice"): + EmptyLite(strategy=strategy) + + +def test_run_input_output(): + """Test that the dynamically patched run() method receives the input arguments and returns the result.""" + + class Lite(LightningLite): + + run_args = () + run_kwargs = {} + + def run(self, *args, **kwargs): + self.run_args = args + self.run_kwargs = kwargs + return "result" + + lite = Lite() + result = lite.run(1, 2, three=3) + assert result == "result" + assert lite.run_args == (1, 2) + assert lite.run_kwargs == {"three": 3} + + +def test_setup_optimizers(): + """Test that setup_optimizers can handle no optimizers, one optimizer, or multiple optimizers.""" + lite = EmptyLite() + model = nn.Linear(1, 2) + optimizer0 = torch.optim.SGD(model.parameters(), lr=0.1) + optimizer1 = torch.optim.Adam(model.parameters(), lr=0.1) + + # no optimizer + lite_model = lite.setup(model) + assert isinstance(lite_model, _LiteModule) + assert lite_model.module is model + + # single optimizer + lite_model, lite_optimizer = lite.setup(model, optimizer0) + assert isinstance(lite_model, _LiteModule) + assert isinstance(lite_optimizer, _LiteOptimizer) + assert lite_model.module is model + assert lite_optimizer.optimizer is optimizer0 + + # multiple optimizers + lite_model, lite_optimizer0, lite_optimizer1 = lite.setup(model, optimizer0, optimizer1) + assert isinstance(lite_model, _LiteModule) + assert isinstance(lite_optimizer0, _LiteOptimizer) + assert isinstance(lite_optimizer1, _LiteOptimizer) + assert lite_model.module is model + assert lite_optimizer0.optimizer is optimizer0 + assert lite_optimizer1.optimizer is optimizer1 + + +def test_setup_twice_fails(): + """Test that calling setup with a model or optimizer that is already wrapped fails.""" + lite = EmptyLite() + model = nn.Linear(1, 2) + optimizer = torch.optim.Adam(model.parameters()) + + lite_model, lite_optimizer = lite.setup(model, optimizer) + with pytest.raises(MisconfigurationException, match="A model should be passed only once to the"): + lite.setup(lite_model, optimizer) + + lite_model, lite_optimizer = lite.setup(model, optimizer) + with pytest.raises(MisconfigurationException, match="An optimizer should be passed only once to the"): + lite.setup(model, lite_optimizer) + + +def test_setup_tracks_num_models(): + """Test that setup() tracks how many times it has setup a model.""" + lite = EmptyLite() + model = nn.Linear(1, 2) + optimizer = torch.optim.Adam(model.parameters()) + + assert lite._num_models == 0 + lite.setup(model, optimizer) + assert lite._num_models == 1 + + lite.setup(model, optimizer) + assert lite._num_models == 2 + + +def test_setup_dataloaders_unsupported_type(): + """Test that the setup_dataloaders method fails when provided with non-DataLoader objects.""" + lite = EmptyLite() + with pytest.raises(MisconfigurationException, match="Only PyTorch DataLoader are currently supported"): + lite.setup_dataloaders(range(2)) # type: ignore + + +def test_setup_dataloaders_return_type(): + """Test that the setup method returns the dataloaders wrapped as LiteDataLoader and in the right order.""" + lite = EmptyLite() + + # single dataloader + lite_dataloader = lite.setup_dataloaders(DataLoader(range(2))) + assert isinstance(lite_dataloader, _LiteDataLoader) + + # multiple dataloaders + dataset0 = Mock() + dataset1 = Mock() + dataloader0 = DataLoader(dataset0) + dataloader1 = DataLoader(dataset1) + lite_dataloader0, lite_dataloader1 = lite.setup_dataloaders(dataloader0, dataloader1) + assert isinstance(lite_dataloader0, _LiteDataLoader) + assert isinstance(lite_dataloader1, _LiteDataLoader) + assert lite_dataloader0.dataset is dataset0 + assert lite_dataloader1.dataset is dataset1 + + +def test_setup_dataloaders_twice_fails(): + """Test that calling setup_dataloaders with a dataloader that is already wrapped fails.""" + lite = EmptyLite() + dataloader = DataLoader(range(2)) + lite_dataloader = lite.setup_dataloaders(dataloader) + + with pytest.raises(MisconfigurationException, match="A dataloader should be passed only once to the"): + lite.setup_dataloaders(lite_dataloader) + + +@mock.patch( + "pytorch_lightning.lite.lite.LightningLite.device", + new_callable=PropertyMock, + return_value=torch.device("cuda", 1), +) +def test_setup_dataloaders_move_to_device(lite_device_mock): + """Test that the setup configures LiteDataLoader to move the data to the device automatically.""" + lite = EmptyLite() + lite_dataloaders = lite.setup_dataloaders(DataLoader(Mock()), DataLoader(Mock()), move_to_device=False) + assert all(dl.device is None for dl in lite_dataloaders) + lite_device_mock.assert_not_called() + + lite = EmptyLite() + lite_dataloaders = lite.setup_dataloaders(DataLoader(Mock()), DataLoader(Mock()), move_to_device=True) + assert all(dl.device == torch.device("cuda", 1) for dl in lite_dataloaders) + lite_device_mock.assert_called() + + +def test_setup_dataloaders_distributed_sampler_not_needed(): + """Test that replace_sampler option has no effect when no distributed sampler is needed.""" + custom_sampler = Mock(spec=Sampler) + dataloader = DataLoader(Mock(), sampler=custom_sampler) + + # keep the custom sampler when not needed to replace + lite = EmptyLite() + lite_dataloader = lite.setup_dataloaders(dataloader, replace_sampler=True) + assert lite_dataloader.sampler is custom_sampler + + +@pytest.mark.parametrize( + "strategy", + [ + DistributedType.DP, + DistributedType.DDP, + DistributedType.DDP_SPAWN, + DistributedType.TPU_SPAWN, + pytest.param(DistributedType.DEEPSPEED, marks=RunIf(deepspeed=True)), + pytest.param(DistributedType.DDP_SHARDED, marks=RunIf(fairscale=True)), + pytest.param(DistributedType.DDP_SHARDED_SPAWN, marks=RunIf(fairscale=True)), + ], +) +def test_setup_dataloaders_replace_custom_sampler(strategy): + """Test that asking to replace a custom sampler results in an error when a distributed sampler would be + needed.""" + custom_sampler = Mock(spec=Sampler) + dataloader = DataLoader(Mock(), sampler=custom_sampler) + + # explicitly asking to replace when a custom sampler is already configured raises an exception + lite = EmptyLite(accelerator="cpu", strategy=strategy, devices=2) + if lite._accelerator_connector.is_distributed: + with pytest.raises(MisconfigurationException, match="You seem to have configured a sampler in your DataLoader"): + lite.setup_dataloaders(dataloader, replace_sampler=True) + + # setting `replace_sampler=False` leaves the sampler untouched + lite_dataloader = lite.setup_dataloaders(dataloader, replace_sampler=False) + assert lite_dataloader.sampler is custom_sampler + + +@pytest.mark.parametrize( + "strategy", + [ + DistributedType.DP, + DistributedType.DDP, + DistributedType.DDP_SPAWN, + DistributedType.TPU_SPAWN, + pytest.param(DistributedType.DEEPSPEED, marks=RunIf(deepspeed=True)), + pytest.param(DistributedType.DDP_SHARDED, marks=RunIf(fairscale=True)), + pytest.param(DistributedType.DDP_SHARDED_SPAWN, marks=RunIf(fairscale=True)), + ], +) +@pytest.mark.parametrize("shuffle", [True, False]) +def test_setup_dataloaders_replace_standard_sampler(shuffle, strategy): + """Test that Lite replaces the default samplers with DistributedSampler automatically.""" + lite = EmptyLite(accelerator="cpu", strategy=strategy, devices=2) + is_distributed = lite._accelerator_connector.is_distributed + lite_dataloader = lite.setup_dataloaders(DataLoader(range(3), shuffle=shuffle)) + assert not is_distributed or isinstance(lite_dataloader.sampler, DistributedSampler) + + +@pytest.mark.parametrize( + "accelerator, expected", + [ + ("cpu", torch.device("cpu")), + pytest.param("gpu", torch.device("cuda", 0), marks=RunIf(min_gpus=1)), + pytest.param("tpu", torch.device("xla", 0), marks=RunIf(tpu=True)), + ], +) +def test_to_device(accelerator, expected): + """Test that the to_device method can move various objects to the device determined by the accelerator.""" + lite = EmptyLite(accelerator=accelerator, devices=1) + + # module + module = torch.nn.Linear(2, 3) + module = lite.to_device(module) + assert all(param.device == expected for param in module.parameters()) + + # tensor + tensor = torch.rand(2, 2) + tensor = lite.to_device(tensor) + assert tensor.device == expected + + # collection + collection = {"data": torch.rand(2, 2), "int": 1} + collection = lite.to_device(collection) + assert collection["data"].device == expected + + +def test_rank_properties(): + """Test that the rank properties are determined by the strategy.""" + lite = EmptyLite() + lite._strategy = Mock(spec=TrainingTypePlugin) + lite._strategy.world_size = 1000 + assert lite.world_size == 1000 + lite._strategy.global_rank = 100 + assert lite.global_rank == 100 + lite._strategy.local_rank = 10 + assert lite.local_rank == 10 + lite._strategy.node_rank = 1 + assert lite.node_rank == 1 + + +def test_backward(): + """Test that backward() calls into the precision plugin.""" + lite = EmptyLite() + lite._precision_plugin = Mock(spec=PrecisionPlugin) + loss = Mock() + lite.backward(loss, "arg", keyword="kwarg") + lite._precision_plugin._run_backward.assert_called_with(loss, None, "arg", keyword="kwarg") + + +@RunIf(deepspeed=True) +def test_backward_model_input_required(): + """Test that when using deepspeed and multiple models, backward() requires the model as input.""" + lite = EmptyLite(strategy="deepspeed") + + model0 = nn.Linear(1, 2) + model1 = nn.Linear(1, 2) + + optimizer0 = torch.optim.Adam(model0.parameters()) + optimizer1 = torch.optim.Adam(model1.parameters()) + + lite._strategy._setup_model_and_optimizer = lambda *args: args + + lite.setup(model0, optimizer0) + lite.setup(model1, optimizer1) + + loss = model0(torch.randn(1, 1)).sum() + + with pytest.raises(MisconfigurationException, match="please provide the model used to perform"): + lite.backward(loss) + + +@RunIf(min_gpus=2, deepspeed=True, special=True) +def test_deepspeed_multiple_models(): + class Lite(LightningLite): + def run(self): + model = BoringModel() + optimizer = torch.optim.SGD(model.parameters(), lr=0.0001) + model, optimizer = self.setup(model, optimizer) + state_dict = deepcopy(model.state_dict()) + + for _ in range(2): + optimizer.zero_grad() + x = model(torch.randn(1, 32).to(self.device)) + loss = x.sum() + self.backward(loss, model=model) + optimizer.step() + + for mw_b, mw_a in zip(state_dict.values(), model.state_dict().values()): + assert not torch.equal(mw_b, mw_a) + + seed_everything(42) + model_1 = BoringModel() + optimizer_1 = torch.optim.SGD(model_1.parameters(), lr=0.0001) + + seed_everything(42) + model_2 = BoringModel() + optimizer_2 = torch.optim.SGD(model_2.parameters(), lr=0.0001) + + for mw_1, mw_2 in zip(model_1.state_dict().values(), model_2.state_dict().values()): + assert torch.equal(mw_1, mw_2) + + model_1, optimizer_1 = self.setup(model_1, optimizer_1) + model_2, optimizer_2 = self.setup(model_2, optimizer_2) + + seed_everything(42) + data_list = [] + for _ in range(2): + optimizer_1.zero_grad() + data = torch.randn(1, 32).to(self.device) + data_list.append(data) + x = model_1(data) + loss = x.sum() + self.backward(loss, model=model_1) + optimizer_1.step() + + for mw_1, mw_2 in zip(model_1.state_dict().values(), model_2.state_dict().values()): + assert not torch.equal(mw_1, mw_2) + + for data in data_list: + optimizer_2.zero_grad() + x = model_2(data) + loss = x.sum() + self.backward(loss, model=model_2) + optimizer_2.step() + + for mw_1, mw_2 in zip(model_1.state_dict().values(), model_2.state_dict().values()): + assert torch.equal(mw_1, mw_2) + + # Verify collectives works as expected + ranks = self.all_gather(torch.tensor([self.local_rank]).to(self.device)) + assert torch.equal(ranks.cpu(), torch.tensor([[0], [1]])) + assert self.broadcast(True) + assert self.is_global_zero == (self.local_rank == 0) + + Lite(strategy=DeepSpeedPlugin(stage=3, logging_batch_size_per_gpu=1), devices=2, accelerator="gpu").run() diff --git a/tests/lite/test_parity.py b/tests/lite/test_parity.py new file mode 100644 index 0000000000000..4b52448ceff71 --- /dev/null +++ b/tests/lite/test_parity.py @@ -0,0 +1,237 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from contextlib import contextmanager +from copy import deepcopy +from functools import partial +from typing import Callable, Generator + +import pytest +import torch +import torch.distributed +import torch.multiprocessing as mp +import torch.nn.functional +from torch import nn +from torch.cuda import is_available +from torch.nn.parallel.distributed import DistributedDataParallel +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler + +from pytorch_lightning import seed_everything +from pytorch_lightning.lite import LightningLite +from pytorch_lightning.plugins.environments.lightning_environment import find_free_network_port +from pytorch_lightning.plugins.training_type.ddp_spawn import DDPSpawnPlugin +from pytorch_lightning.utilities.apply_func import apply_to_collection, move_data_to_device +from pytorch_lightning.utilities.cloud_io import atomic_save +from pytorch_lightning.utilities.imports import _TORCH_GREATER_EQUAL_DEV_1_10 +from tests.helpers.boring_model import RandomDataset +from tests.helpers.runif import RunIf + + +class BoringModel(nn.Module): + def __init__(self): + super().__init__() + self.layer = torch.nn.Linear(32, 2, bias=False) + + def forward(self, x): + x = self.layer(x) + return torch.nn.functional.mse_loss(x, torch.ones_like(x)) + + +def configure_optimizers(module: nn.Module): + return torch.optim.SGD(module.parameters(), lr=0.0001) + + +def main( + move_to_device: Callable, + model: nn.Module, + train_dataloader: DataLoader, + num_epochs: int = 10, +): + model = move_to_device(model) + optimizer = configure_optimizers(model) + + for _ in range(num_epochs): + model.train() + for batch in train_dataloader: + batch = move_to_device(batch) + optimizer.zero_grad() + loss = model(batch) + loss.backward() + optimizer.step() + + return model.state_dict() + + +class LiteRunner(LightningLite): + def run(self, model: nn.Module, train_dataloader: DataLoader, num_epochs: int = 10, tmpdir: str = None): + optimizer = configure_optimizers(model) + model, optimizer = self.setup(model, optimizer) + train_dataloader = self.setup_dataloaders(train_dataloader) + + model.train() + for _ in range(num_epochs): + for batch in train_dataloader: + batch = self.to_device(batch) + optimizer.zero_grad() + loss = model(batch) + self.backward(loss) + optimizer.step() + + if isinstance(self._strategy, DDPSpawnPlugin) and tmpdir and self.global_rank == 0: + checkpoint_path = os.path.join(tmpdir, "model.pt") + atomic_save(model.state_dict(), checkpoint_path) + return checkpoint_path + + +@contextmanager +def precision_context(precision, accelerator) -> Generator[None, None, None]: + if precision == 32: + yield + return + if precision == 16 and accelerator == "gpu": + with torch.cuda.amp.autocast(): + yield + elif accelerator == "cpu": + with torch.cpu.amp.autocast(dtype=torch.float16 if precision == 16 else torch.bfloat16): + yield + else: + with torch.cuda.amp.autocast(): + yield + + +@pytest.mark.parametrize( + "precision, strategy, devices, accelerator", + [ + pytest.param(32, None, 1, "cpu"), + pytest.param(32, None, 1, "gpu", marks=pytest.mark.skipif(not is_available(), reason="requires a GPU")), + pytest.param(16, None, 1, "gpu", marks=pytest.mark.skipif(not is_available(), reason="requires a GPU")), + pytest.param( + "bf16", + None, + 1, + "gpu", + marks=pytest.mark.skipif( + not (_TORCH_GREATER_EQUAL_DEV_1_10 and is_available()), + reason="bfloat16 and requires GPU isn't available.", + ), + ), + ], +) +def test_boring_lite_model_single_device(precision, strategy, devices, accelerator, tmpdir): + seed_everything(42) + train_dataloader = DataLoader(RandomDataset(32, 8)) + model = BoringModel() + num_epochs = 1 + state_dict = deepcopy(model.state_dict()) + + lite = LiteRunner(precision=precision, strategy=strategy, devices=devices, accelerator=accelerator) + lite.run(model, train_dataloader, num_epochs=num_epochs) + lite_state_dict = model.state_dict() + + with precision_context(precision, accelerator): + model.load_state_dict(state_dict) + pure_state_dict = main(lite.to_device, model, train_dataloader, num_epochs=num_epochs) + + state_dict = apply_to_collection(state_dict, torch.Tensor, lite.to_device) + for w_pure, w_lite in zip(state_dict.values(), lite_state_dict.values()): + assert not torch.equal(w_pure, w_lite) + + for w_pure, w_lite in zip(pure_state_dict.values(), lite_state_dict.values()): + assert torch.equal(w_pure, w_lite) + + +def run(rank, model, train_dataloader, num_epochs, precision, accelerator, tmpdir): + os.environ["LOCAL_RANK"] = str(rank) + if torch.distributed.is_available() and not torch.distributed.is_initialized(): + torch.distributed.init_process_group("gloo", rank=rank, world_size=2) + + to_device = partial(move_data_to_device, device=torch.device("cuda", rank)) + model = DistributedDataParallel( + to_device(model), + device_ids=[rank], + ) + train_dataloader = DataLoader( + train_dataloader.dataset, + sampler=DistributedSampler(train_dataloader.dataset, rank=rank, num_replicas=2, seed=42, drop_last=False), + ) + with precision_context(precision, accelerator): + main(to_device, model, train_dataloader, num_epochs=num_epochs) + + if rank == 0: + atomic_save(model.state_dict(), os.path.join(tmpdir, "model_spawn.pt")) + + +# @pytest.mark.skipif(True, reason="Skipping as it takes 80 seconds.") +@RunIf(min_gpus=2) +@pytest.mark.parametrize( + "precision, strategy, devices, accelerator", + [ + (32, "ddp_spawn", 2, "gpu"), + ], +) +def test_boring_lite_model_ddp_spawn(precision, strategy, devices, accelerator, tmpdir): + seed_everything(42) + train_dataloader = DataLoader(RandomDataset(32, 8)) + model = BoringModel() + num_epochs = 1 + state_dict = deepcopy(model.state_dict()) + + lite = LiteRunner(precision=precision, strategy=strategy, devices=devices, accelerator=accelerator) + checkpoint_path = lite.run(model, train_dataloader, num_epochs=num_epochs, tmpdir=tmpdir) + spawn_model_state_dict = torch.load(checkpoint_path) + + for w_pure, w_lite in zip(state_dict.values(), spawn_model_state_dict.values()): + assert not torch.equal(w_pure.cpu(), w_lite.cpu()) + + model.load_state_dict(state_dict) + os.environ["MASTER_ADDR"] = "127.0.0.1" + os.environ["MASTER_PORT"] = str(find_free_network_port()) + mp.spawn(run, args=(model, train_dataloader, num_epochs, precision, accelerator, tmpdir), nprocs=2) + spawn_pure_model_state_dict = torch.load(os.path.join(tmpdir, "model_spawn.pt")) + + for w_pure, w_lite in zip(spawn_pure_model_state_dict.values(), spawn_model_state_dict.values()): + assert torch.equal(w_pure.cpu(), w_lite.cpu()) + + +@RunIf(min_gpus=2, special=True) +@pytest.mark.parametrize( + "precision, strategy, devices, accelerator", + [ + (32, "ddp", 2, "gpu"), + ], +) +def test_boring_lite_model_ddp(precision, strategy, devices, accelerator, tmpdir): + seed_everything(42) + train_dataloader = DataLoader(RandomDataset(32, 4)) + model = BoringModel() + num_epochs = 1 + state_dict = deepcopy(model.state_dict()) + + lite = LiteRunner(precision=precision, strategy=strategy, devices=devices, accelerator=accelerator) + lite.run(model, train_dataloader, num_epochs=num_epochs, tmpdir=tmpdir) + + lite_model_state_dict = model.state_dict() + + for w_pure, w_lite in zip(state_dict.values(), lite_model_state_dict.values()): + assert not torch.equal(w_pure.cpu(), w_lite.cpu()) + + seed_everything(42) + train_dataloader = DataLoader(RandomDataset(32, 4)) + model = BoringModel() + run(lite.global_rank, model, train_dataloader, num_epochs, precision, accelerator, tmpdir) + pure_model_state_dict = model.state_dict() + + for w_pure, w_lite in zip(pure_model_state_dict.values(), lite_model_state_dict.values()): + assert torch.equal(w_pure.cpu(), w_lite.cpu()) diff --git a/tests/lite/test_wrappers.py b/tests/lite/test_wrappers.py new file mode 100644 index 0000000000000..faed290b75629 --- /dev/null +++ b/tests/lite/test_wrappers.py @@ -0,0 +1,106 @@ +# Copyright The PyTorch Lightning team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from unittest.mock import ANY, Mock + +import pytest +import torch + +from pytorch_lightning.lite import LightningLite +from pytorch_lightning.lite.wrappers import _LiteDataLoader, _LiteModule, _LiteOptimizer +from tests.helpers.runif import RunIf + + +class EmptyLite(LightningLite): + def run(self): + pass + + +def test_lite_module_wraps(): + """Test that the wrapped module is accessible via the property.""" + module = Mock() + assert _LiteModule(module, Mock()).module is module + + +@RunIf(min_gpus=1) +@pytest.mark.parametrize( + "precision, input_type, expected_type", + [ + (32, torch.float16, torch.float32), + (32, torch.float32, torch.float32), + (32, torch.float64, torch.float32), + (16, torch.float32, torch.float16), + (16, torch.float64, torch.float16), + # ("mixed", torch.float32, torch.float16), # TODO: support precision="mixed" + ], +) +def test_lite_module_forward_conversion(precision, input_type, expected_type): + """Test that the LiteModule performs autocasting on the input tensors and during forward().""" + lite = EmptyLite(precision=precision, accelerator="gpu", devices=1) + device = torch.device("cuda", 0) + + def check_autocast(forward_input): + assert precision not in (16, "mixed") or torch.is_autocast_enabled() + return forward_input + + module = Mock(wraps=torch.nn.Linear(1, 1), side_effect=check_autocast) + lite_module = _LiteModule(module, lite._accelerator).to(device) + out = lite_module(torch.rand(1, dtype=input_type, device=device)) + assert module.call_args[0][0].dtype == expected_type + assert out.dtype == torch.get_default_dtype() + + +@pytest.mark.parametrize( + "src_device, dest_device", + [ + (torch.device("cpu"), torch.device("cpu")), + pytest.param(torch.device("cpu"), torch.device("cuda", 0), marks=RunIf(min_gpus=1)), + pytest.param(torch.device("cuda", 0), torch.device("cpu"), marks=RunIf(min_gpus=1)), + ], +) +def test_lite_dataloader_device_placement(src_device, dest_device): + """Test that the LiteDataLoader moves data to the device in its iterator.""" + sample0 = torch.tensor(0, device=src_device) + sample1 = torch.tensor(1, device=src_device) + sample2 = {"data": torch.tensor(2, device=src_device)} + sample3 = {"data": torch.tensor(3, device=src_device)} + data = [sample0, sample1, sample2, sample3] + lite_dataloader = _LiteDataLoader(device=dest_device, dataset=data, batch_size=2) + iterator = iter(lite_dataloader) + + batch0 = next(iterator) + assert torch.equal(batch0, torch.tensor([0, 1], device=dest_device)) + + batch1 = next(iterator) + assert torch.equal(batch1["data"], torch.tensor([2, 3], device=dest_device)) + + +def test_lite_optimizer_wraps(): + """Test that the LiteOptimizer fully wraps the optimizer.""" + optimizer_cls = torch.optim.SGD + optimizer = Mock(spec=optimizer_cls) + lite_optimizer = _LiteOptimizer(optimizer, Mock()) + assert lite_optimizer.optimizer is optimizer + assert isinstance(lite_optimizer, optimizer_cls) + + +def test_lite_optimizer_steps(): + """Test that the LiteOptimizer forwards the step() and zero_grad() calls to the wrapped optimizer.""" + optimizer = Mock() + accelerator = Mock() + lite_optimizer = _LiteOptimizer(optimizer=optimizer, accelerator=accelerator) + lite_optimizer.step() + accelerator.optimizer_step.assert_called_once() + accelerator.optimizer_step.assert_called_with(optimizer, opt_idx=0, lambda_closure=ANY, model=accelerator.model) + lite_optimizer.zero_grad() + optimizer.zero_grad.assert_called_once() From be390986809d39f6a509e6847f44bfa32c31c2de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 27 Oct 2021 09:59:06 +0200 Subject: [PATCH 238/331] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e145012a2c914..462c4dc1f70ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -220,7 +220,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). * Implemented `DeepSpeedPlugin._setup_model_and_optimizers` ([#10009](https://github.com/PyTorchLightning/pytorch-lightning/pull/10009), [#10064](https://github.com/PyTorchLightning/pytorch-lightning/pull/10064)) * Implemented `{DDPShardedPlugin,DDPShardedSpawnPlugin}._setup_model_and_optimizers` ([#10028](https://github.com/PyTorchLightning/pytorch-lightning/pull/10028), [#10064](https://github.com/PyTorchLightning/pytorch-lightning/pull/10064)) * Added optional `model` argument to the `optimizer_step` methods in accelerators and plugins ([#10023](https://github.com/PyTorchLightning/pytorch-lightning/pull/10023)) - + * Added `pytorch_lightning.lite` package ([#?](https://github.com/PyTorchLightning/pytorch-lightning/pull/?)) - Added `XLACheckpointIO` plugin ([#9972](https://github.com/PyTorchLightning/pytorch-lightning/pull/9972)) From 824c11d838245406093cf681f4b50a209d4ead1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Wed, 27 Oct 2021 12:01:30 +0200 Subject: [PATCH 239/331] update test to ensure spawn result --- tests/lite/test_lite.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index def9ce29ac9dc..a090a4926658d 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -59,24 +59,24 @@ def test_unsupported_strategy(strategy): EmptyLite(strategy=strategy) -def test_run_input_output(): - """Test that the dynamically patched run() method receives the input arguments and returns the result.""" - - class Lite(LightningLite): +class LiteReturnSpawnResult(LightningLite): + def run(self, *args, **kwargs): + return args, kwargs, "result", self.local_rank - run_args = () - run_kwargs = {} - def run(self, *args, **kwargs): - self.run_args = args - self.run_kwargs = kwargs - return "result" - - lite = Lite() +@pytest.mark.parametrize( + "accelerator, strategy, devices", + [ + ("cpu", None, None), + ("cpu", "ddp_spawn", 2), + pytest.param("tpu", "tpu_spawn", 1, marks=RunIf(tpu=True)), + ], +) +def test_run_input_output(accelerator, strategy, devices): + """Test that the dynamically patched run() method receives the input arguments and returns the result.""" + lite = LiteReturnSpawnResult(accelerator=accelerator, strategy=strategy, devices=devices) result = lite.run(1, 2, three=3) - assert result == "result" - assert lite.run_args == (1, 2) - assert lite.run_kwargs == {"three": 3} + assert result == ((1, 2), {"three": 3}, "result", 0) def test_setup_optimizers(): From 81636fe31bfff97adc7bd253fae3c32156642cac Mon Sep 17 00:00:00 2001 From: Kaushik B Date: Wed, 27 Oct 2021 14:26:01 +0530 Subject: [PATCH 240/331] Add sleep to fix the rendezous error --- pytorch_lightning/plugins/training_type/tpu_spawn.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pytorch_lightning/plugins/training_type/tpu_spawn.py b/pytorch_lightning/plugins/training_type/tpu_spawn.py index 0de53f7b61503..3ac4c7a961ad3 100644 --- a/pytorch_lightning/plugins/training_type/tpu_spawn.py +++ b/pytorch_lightning/plugins/training_type/tpu_spawn.py @@ -277,7 +277,11 @@ def _wrapped_function( result = function(*args, **kwargs) if self.local_rank == 0: return_queue.put(move_data_to_device(result, "cpu")) + self.barrier("end-process") + # https://github.com/pytorch/xla/issues/2190#issuecomment-641665358 + if self.local_rank == 0: + time.sleep(2) def _worker_setup(self, process_idx: int): reset_seed() From e45f73661093a3ac5436cc66b86c25a2c9c3593a Mon Sep 17 00:00:00 2001 From: tchaton Date: Wed, 27 Oct 2021 17:28:27 +0100 Subject: [PATCH 241/331] update --- tests/lite/test_parity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/lite/test_parity.py b/tests/lite/test_parity.py index 4b52448ceff71..48ed2bb22cd98 100644 --- a/tests/lite/test_parity.py +++ b/tests/lite/test_parity.py @@ -34,7 +34,7 @@ from pytorch_lightning.plugins.training_type.ddp_spawn import DDPSpawnPlugin from pytorch_lightning.utilities.apply_func import apply_to_collection, move_data_to_device from pytorch_lightning.utilities.cloud_io import atomic_save -from pytorch_lightning.utilities.imports import _TORCH_GREATER_EQUAL_DEV_1_10 +from pytorch_lightning.utilities.imports import _TORCH_GREATER_EQUAL_1_10 from tests.helpers.boring_model import RandomDataset from tests.helpers.runif import RunIf @@ -123,7 +123,7 @@ def precision_context(precision, accelerator) -> Generator[None, None, None]: 1, "gpu", marks=pytest.mark.skipif( - not (_TORCH_GREATER_EQUAL_DEV_1_10 and is_available()), + not (_TORCH_GREATER_EQUAL_1_10 and is_available()), reason="bfloat16 and requires GPU isn't available.", ), ), From 0decebae66341d9035faf9fa5ca27a12282381dd Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Wed, 27 Oct 2021 19:20:37 +0200 Subject: [PATCH 242/331] Docstrings and CHANGELOG --- CHANGELOG.md | 2 +- pytorch_lightning/lite/lite.py | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 462c4dc1f70ce..5f3ca024aeddc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -220,7 +220,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). * Implemented `DeepSpeedPlugin._setup_model_and_optimizers` ([#10009](https://github.com/PyTorchLightning/pytorch-lightning/pull/10009), [#10064](https://github.com/PyTorchLightning/pytorch-lightning/pull/10064)) * Implemented `{DDPShardedPlugin,DDPShardedSpawnPlugin}._setup_model_and_optimizers` ([#10028](https://github.com/PyTorchLightning/pytorch-lightning/pull/10028), [#10064](https://github.com/PyTorchLightning/pytorch-lightning/pull/10064)) * Added optional `model` argument to the `optimizer_step` methods in accelerators and plugins ([#10023](https://github.com/PyTorchLightning/pytorch-lightning/pull/10023)) - * Added `pytorch_lightning.lite` package ([#?](https://github.com/PyTorchLightning/pytorch-lightning/pull/?)) + * Added `pytorch_lightning.lite` package ([#10175](https://github.com/PyTorchLightning/pytorch-lightning/pull/10175)) - Added `XLACheckpointIO` plugin ([#9972](https://github.com/PyTorchLightning/pytorch-lightning/pull/9972)) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 49798b138567e..0394b6d1d7884 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -16,7 +16,7 @@ from contextlib import contextmanager from functools import partial from pathlib import Path -from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, overload, Sequence, Tuple, Union +from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, Sequence, Tuple, Union import torch import torch.nn as nn @@ -46,21 +46,22 @@ class LightningLite(ABC): """Lite accelerates your PyTorch training or inference code with minimal changes required. - - Automatic placement of models and data onto the device - - Automatic support for mixed and double precision (smaller memory footprint) + - Automatic placement of models and data onto the device. + - Automatic support for mixed and double precision (smaller memory footprint). - Seamless switching between hardware (CPU, GPU, TPU) and distributed training strategies - (data-parallel training, sharded training, etc.) - - Automated spawning of processes, no launch utilities required - - Multi-node support + (data-parallel training, sharded training, etc.). + - Automated spawning of processes, no launch utilities required. + - Multi-node support. Args: - accelerator: The hardware to run on. Possible choices are: cpu, gpu, tpu, auto. + accelerator: The hardware to run on. Possible choices are: ```cpu"``, ```gpu"``, ```tpu"``, ```auto"``. strategy: Strategy for how to run across multiple devices. Possible choices are: - dp, ddp, ddp_spawn, tpu_spawn, deepspeed, ddp_sharded. - devices: Number of devices to train on (int) or which GPUs to train on (list or str). The value applies - per node. + ```dp"``, ```ddp"``, ```ddp_spawn"``, ```deepspeed"``, ```ddp_sharded"``. + devices: Number of devices to train on (``int``) or which GPUs to train on (``list`` or ``str``). + The value applies per node. num_nodes: Number of GPU nodes for distributed training. - precision: Double precision (64), full precision (32), half precision (16) or bfloat16 precision (bf16). + precision: Double precision (``64``), full precision (``32``), half precision (``16``), + or bfloat16 precision (```bf16"``). plugins: One or several custom plugins gpus: Provides the same function as the ``devices`` argument but implies ``accelerator="gpu"``. tpu_cores: Provides the same function as the ``devices`` argument but implies ``accelerator="tpu"``. @@ -250,7 +251,7 @@ def backward(self, tensor: Tensor, *args: Any, model: Optional[_LiteModule] = No **kwargs: Optional named keyword arguments passed to the underlying backward function. Note: - When using ``strategy='deepspeed'`` and multiple models were setup, it is required to pass in the + When using ``strategy='deepspeed"`` and multiple models were setup, it is required to pass in the model as argument here. """ module = model.module if model is not None else model From 5d14e832bae5c4a43c961e298541107331425bcf Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Wed, 27 Oct 2021 19:24:08 +0200 Subject: [PATCH 243/331] Fixes to previous commit. Mention devices=auto (not yet implemented). Remove tpu spawn --- pytorch_lightning/lite/lite.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 0394b6d1d7884..075b3ad729b36 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -54,14 +54,14 @@ class LightningLite(ABC): - Multi-node support. Args: - accelerator: The hardware to run on. Possible choices are: ```cpu"``, ```gpu"``, ```tpu"``, ```auto"``. + accelerator: The hardware to run on. Possible choices are: ``"cpu"``, ``"gpu"``, ``"tpu"``, ``"auto"``. strategy: Strategy for how to run across multiple devices. Possible choices are: - ```dp"``, ```ddp"``, ```ddp_spawn"``, ```deepspeed"``, ```ddp_sharded"``. - devices: Number of devices to train on (``int``) or which GPUs to train on (``list`` or ``str``). + ``"dp"``, ``"ddp"``, ``"ddp_spawn"``, ``"deepspeed"``, ``"ddp_sharded"``. + devices: Number of devices to train on (``int``), which GPUs to train on (``list`` or ``str``), or ``"auto"``. The value applies per node. num_nodes: Number of GPU nodes for distributed training. precision: Double precision (``64``), full precision (``32``), half precision (``16``), - or bfloat16 precision (```bf16"``). + or bfloat16 precision (``"bf16"``). plugins: One or several custom plugins gpus: Provides the same function as the ``devices`` argument but implies ``accelerator="gpu"``. tpu_cores: Provides the same function as the ``devices`` argument but implies ``accelerator="tpu"``. @@ -448,7 +448,6 @@ def _supported_strategy_types() -> Sequence[str]: DistributedType.DP, DistributedType.DDP, DistributedType.DDP_SPAWN, - DistributedType.TPU_SPAWN, DistributedType.DEEPSPEED, DistributedType.DDP_SHARDED, DistributedType.DDP_SHARDED_SPAWN, From 11862e869e04c121c71c2268320382c8819a0f48 Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Wed, 27 Oct 2021 22:34:29 +0200 Subject: [PATCH 244/331] Fix test --- tests/lite/test_lite.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index def9ce29ac9dc..b9508a64ec0e4 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -209,7 +209,6 @@ def test_setup_dataloaders_distributed_sampler_not_needed(): DistributedType.DP, DistributedType.DDP, DistributedType.DDP_SPAWN, - DistributedType.TPU_SPAWN, pytest.param(DistributedType.DEEPSPEED, marks=RunIf(deepspeed=True)), pytest.param(DistributedType.DDP_SHARDED, marks=RunIf(fairscale=True)), pytest.param(DistributedType.DDP_SHARDED_SPAWN, marks=RunIf(fairscale=True)), From ffed5ced1c791f714c01b44ba6f42dc30f532179 Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Wed, 27 Oct 2021 22:34:29 +0200 Subject: [PATCH 245/331] Fix test --- tests/lite/test_lite.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index def9ce29ac9dc..f47f9f1df1434 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -209,7 +209,6 @@ def test_setup_dataloaders_distributed_sampler_not_needed(): DistributedType.DP, DistributedType.DDP, DistributedType.DDP_SPAWN, - DistributedType.TPU_SPAWN, pytest.param(DistributedType.DEEPSPEED, marks=RunIf(deepspeed=True)), pytest.param(DistributedType.DDP_SHARDED, marks=RunIf(fairscale=True)), pytest.param(DistributedType.DDP_SHARDED_SPAWN, marks=RunIf(fairscale=True)), @@ -238,7 +237,6 @@ def test_setup_dataloaders_replace_custom_sampler(strategy): DistributedType.DP, DistributedType.DDP, DistributedType.DDP_SPAWN, - DistributedType.TPU_SPAWN, pytest.param(DistributedType.DEEPSPEED, marks=RunIf(deepspeed=True)), pytest.param(DistributedType.DDP_SHARDED, marks=RunIf(fairscale=True)), pytest.param(DistributedType.DDP_SHARDED_SPAWN, marks=RunIf(fairscale=True)), From c614cf0c3d7a252bfd01ff8516f9f42f1c1b055a Mon Sep 17 00:00:00 2001 From: thomas chaton Date: Thu, 28 Oct 2021 01:11:51 +0100 Subject: [PATCH 246/331] Improve Lite Examples (#10195) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrian Wälchli Co-authored-by: Carlos Mocholí --- docs/source/advanced/mixed_precision.rst | 4 +- docs/source/conf.py | 2 +- pl_examples/basic_examples/README.md | 17 +- .../basic_examples/mnist_examples/README.md | 15 +- .../image_classifier_1_pytorch.py | 146 +++++++++--------- .../mnist_examples/image_classifier_2_lite.py | 127 ++++++++------- .../image_classifier_3_lite_to_lightning.py | 96 ++++++------ .../image_classifier_4_lightning.py | 7 +- ...image_classifier_5_lightning_datamodule.py | 3 +- pytorch_lightning/callbacks/quantization.py | 6 +- pytorch_lightning/core/lightning.py | 4 +- .../plugins/precision/native_amp.py | 8 +- pytorch_lightning/utilities/__init__.py | 1 - pytorch_lightning/utilities/imports.py | 2 +- tests/core/test_lightning_module.py | 6 +- tests/models/test_amp.py | 6 +- tests/plugins/test_amp_plugins.py | 5 +- 17 files changed, 216 insertions(+), 239 deletions(-) diff --git a/docs/source/advanced/mixed_precision.rst b/docs/source/advanced/mixed_precision.rst index 1c98f663ed5f3..9889c05db243d 100644 --- a/docs/source/advanced/mixed_precision.rst +++ b/docs/source/advanced/mixed_precision.rst @@ -50,14 +50,14 @@ BFloat16 Mixed precision is similar to FP16 mixed precision, however we maintain Since BFloat16 is more stable than FP16 during training, we do not need to worry about any gradient scaling or nan gradient values that comes with using FP16 mixed precision. .. testcode:: - :skipif: not _TORCH_GREATER_EQUAL_DEV_1_10 or not torch.cuda.is_available() + :skipif: not _TORCH_GREATER_EQUAL_1_10 or not torch.cuda.is_available() Trainer(gpus=1, precision="bf16") It is also possible to use BFloat16 mixed precision on the CPU, relying on MKLDNN under the hood. .. testcode:: - :skipif: not _TORCH_GREATER_EQUAL_DEV_1_10 + :skipif: not _TORCH_GREATER_EQUAL_1_10 Trainer(precision="bf16") diff --git a/docs/source/conf.py b/docs/source/conf.py index cbd7a51fa1238..845b3b946972a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -377,7 +377,7 @@ def package_list_from_file(file): _XLA_AVAILABLE, _TPU_AVAILABLE, _TORCHVISION_AVAILABLE, - _TORCH_GREATER_EQUAL_DEV_1_10, + _TORCH_GREATER_EQUAL_1_10, _module_available, ) _JSONARGPARSE_AVAILABLE = _module_available("jsonargparse") diff --git a/pl_examples/basic_examples/README.md b/pl_examples/basic_examples/README.md index cda779c459ad6..b58632cf51158 100644 --- a/pl_examples/basic_examples/README.md +++ b/pl_examples/basic_examples/README.md @@ -6,7 +6,7 @@ Use these examples to test how Lightning works. 5 MNIST examples showing how to gradually convert from pure PyTorch to PyTorch Lightning. -The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst) from pure PyTorch is optional but it might helpful to learn about it. +The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst) from pure PyTorch is optional but it might be helpful to learn about it. #### 1 . Image Classifier with Vanilla PyTorch @@ -21,7 +21,7 @@ ______________________________________________________________________ #### 2. Image Classifier with LightningLite -Trains a simple CNN over MNIST using [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst). +This script shows you how to scale the previous script to enable GPU and multi GPU training using [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst). ```bash # cpu / multiple gpus if available @@ -30,7 +30,10 @@ python mnist_examples/image_classifier_2_lite.py ______________________________________________________________________ -Trains a simple CNN over MNIST where `LightningLite` is almost a `LightningModule`. +#### 3. Image Classifier - Conversion Lite to Lightning + +This script shows you to prepare your conversion from [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst) +to `LightningModule`. ```bash # cpu / multiple gpus if available @@ -41,7 +44,7 @@ ______________________________________________________________________ #### 4. Image Classifier with LightningModule -Trains a simple CNN over MNIST with `Lightning Trainer` and the converted `LightningModule`. +This script shows you how the result of the conversion to the `LightningModule` and finally get all the benefits from Lightning. ```bash # cpu @@ -55,7 +58,7 @@ ______________________________________________________________________ #### 5. Image Classifier with LightningModule + LightningDataModule -Trains a simple CNN over MNIST with `Lightning Trainer` and the converted `LightningModule` and `LightningDataModule` +This script shows you how extracts the data related components to a `LightningDataModule`. ```bash # cpu @@ -64,8 +67,8 @@ python mnist_examples/image_classifier_5_lightning_datamodule.py # gpus (any number) python mnist_examples/image_classifier_5_lightning_datamodule.py --trainer.gpus 2 -# Distributed Data Parallel -python backbone_image_classifier.py --trainer.gpus 2 --trainer.accelerator ddp +# data parallel +python mnist_examples/image_classifier_5_lightning_datamodule.py --trainer.gpus 2 --trainer.accelerator 'dp' ``` ______________________________________________________________________ diff --git a/pl_examples/basic_examples/mnist_examples/README.md b/pl_examples/basic_examples/mnist_examples/README.md index 323273d9ff718..68028f7059c6a 100644 --- a/pl_examples/basic_examples/mnist_examples/README.md +++ b/pl_examples/basic_examples/mnist_examples/README.md @@ -2,7 +2,7 @@ 5 MNIST examples showing how to gradually convert from pure PyTorch to PyTorch Lightning. -The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst) from pure PyTorch is optional but it might helpful to learn about it. +The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst) from pure PyTorch is optional but it might be helpful to learn about it. #### 1 . Image Classifier with Vanilla PyTorch @@ -17,7 +17,7 @@ ______________________________________________________________________ #### 2. Image Classifier with LightningLite -Trains a simple CNN over MNIST using [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst). +This script shows you how to scale the previous script to enable GPU and multi GPU training using [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst). ```bash # cpu / multiple gpus if available @@ -28,7 +28,8 @@ ______________________________________________________________________ #### 3. Image Classifier - Conversion Lite to Lightning -Trains a simple CNN over MNIST where `LightningLite` is almost a `LightningModule`. +This script shows you to prepare your conversion from [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst) +to `LightningModule`. ```bash # cpu / multiple gpus if available @@ -39,21 +40,21 @@ ______________________________________________________________________ #### 4. Image Classifier with LightningModule -Trains a simple CNN over MNIST with `Lightning Trainer` and the converted `LightningModule`. +This script shows you how the result of the conversion to the `LightningModule` and finally get all the benefits from Lightning. ```bash # cpu -python mnist_examples/image_classifier_4_lightning.py +python image_classifier_4_lightning.py # gpus (any number) -python mnist_examples/image_classifier_4_lightning.py --trainer.gpus 2 +python image_classifier_4_lightning.py --trainer.gpus 2 ``` ______________________________________________________________________ #### 5. Image Classifier with LightningModule + LightningDataModule -Trains a simple CNN over MNIST with `Lightning Trainer` and the converted `LightningModule` and `LightningDataModule` +This script shows you how extracts the data related components to a `LightningDataModule`. ```bash # cpu diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_1_pytorch.py b/pl_examples/basic_examples/mnist_examples/image_classifier_1_pytorch.py index e7449473194ed..4073c485e6017 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_1_pytorch.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_1_pytorch.py @@ -52,50 +52,80 @@ def forward(self, x): return output -def train(args, model, device, train_loader, optimizer, epoch): - model.train() - for batch_idx, (data, target) in enumerate(train_loader): - data, target = data.to(device), target.to(device) - optimizer.zero_grad() - output = model(data) - loss = F.nll_loss(output, target) - loss.backward() - optimizer.step() - if (batch_idx == 0) or ((batch_idx + 1) % args.log_interval == 0): - print( - "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format( - epoch, - batch_idx * len(data), - len(train_loader.dataset), - 100.0 * batch_idx / len(train_loader), - loss.item(), - ) - ) - if args.dry_run: - break +def run(hparams): + + torch.manual_seed(hparams.seed) + + use_cuda = torch.cuda.is_available() + device = torch.device("cuda" if use_cuda else "cpu") + + transform = T.Compose([T.ToTensor(), T.Normalize((0.1307,), (0.3081,))]) + train_dataset = MNIST("./data", train=True, download=True, transform=transform) + test_dataset = MNIST("./data", train=False, transform=transform) + train_loader = torch.utils.data.DataLoader( + train_dataset, + batch_size=hparams.batch_size, + ) + test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=hparams.batch_size) + model = Net().to(device) + optimizer = optim.Adadelta(model.parameters(), lr=hparams.lr) + + scheduler = StepLR(optimizer, step_size=1, gamma=hparams.gamma) + + # EPOCH LOOP + for epoch in range(1, hparams.epochs + 1): -def test(args, model, device, test_loader): - model.eval() - test_loss = 0 - correct = 0 - with torch.no_grad(): - for data, target in test_loader: + # TRAINING LOOP + model.train() + for batch_idx, (data, target) in enumerate(train_loader): data, target = data.to(device), target.to(device) + optimizer.zero_grad() output = model(data) - test_loss += F.nll_loss(output, target, reduction="sum").item() # sum up batch loss - pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability - correct += pred.eq(target.view_as(pred)).sum().item() - if args.dry_run: - break - - test_loss /= len(test_loader.dataset) + loss = F.nll_loss(output, target) + loss.backward() + optimizer.step() + if (batch_idx == 0) or ((batch_idx + 1) % hparams.log_interval == 0): + print( + "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format( + epoch, + batch_idx * len(data), + len(train_loader.dataset), + 100.0 * batch_idx / len(train_loader), + loss.item(), + ) + ) + if hparams.dry_run: + break + scheduler.step() - print( - "\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n".format( - test_loss, correct, len(test_loader.dataset), 100.0 * correct / len(test_loader.dataset) + # TESTING LOOP + model.eval() + test_loss = 0 + correct = 0 + with torch.no_grad(): + for data, target in test_loader: + data, target = data.to(device), target.to(device) + output = model(data) + test_loss += F.nll_loss(output, target, reduction="sum").item() # sum up batch loss + pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability + correct += pred.eq(target.view_as(pred)).sum().item() + if hparams.dry_run: + break + + test_loss /= len(test_loader.dataset) + + print( + "\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n".format( + test_loss, correct, len(test_loader.dataset), 100.0 * correct / len(test_loader.dataset) + ) ) - ) + + if hparams.dry_run: + break + + if hparams.save_model: + torch.save(model.state_dict(), "mnist_cnn.pt") def main(): @@ -103,13 +133,9 @@ def main(): parser.add_argument( "--batch-size", type=int, default=64, metavar="N", help="input batch size for training (default: 64)" ) - parser.add_argument( - "--test-batch-size", type=int, default=1000, metavar="N", help="input batch size for testing (default: 1000)" - ) parser.add_argument("--epochs", type=int, default=14, metavar="N", help="number of epochs to train (default: 14)") parser.add_argument("--lr", type=float, default=1.0, metavar="LR", help="learning rate (default: 1.0)") parser.add_argument("--gamma", type=float, default=0.7, metavar="M", help="Learning rate step gamma (default: 0.7)") - parser.add_argument("--no-cuda", action="store_true", default=False, help="disables CUDA training") parser.add_argument("--dry-run", action="store_true", default=False, help="quickly check a single pass") parser.add_argument("--seed", type=int, default=1, metavar="S", help="random seed (default: 1)") parser.add_argument( @@ -120,40 +146,8 @@ def main(): help="how many batches to wait before logging training status", ) parser.add_argument("--save-model", action="store_true", default=False, help="For Saving the current Model") - args = parser.parse_args() - use_cuda = not args.no_cuda and torch.cuda.is_available() - - torch.manual_seed(args.seed) - - device = torch.device("cuda" if use_cuda else "cpu") - - train_kwargs = {"batch_size": args.batch_size} - test_kwargs = {"batch_size": args.test_batch_size} - if use_cuda: - cuda_kwargs = {"num_workers": 1, "pin_memory": True, "shuffle": True} - train_kwargs.update(cuda_kwargs) - test_kwargs.update(cuda_kwargs) - - transform = T.Compose([T.ToTensor(), T.Normalize((0.1307,), (0.3081,))]) - train_dataset = MNIST("./data", train=True, download=True, transform=transform) - test_dataset = MNIST("./data", train=False, transform=transform) - train_loader = torch.utils.data.DataLoader(train_dataset, **train_kwargs) - test_loader = torch.utils.data.DataLoader(test_dataset, **test_kwargs) - - model = Net().to(device) - optimizer = optim.Adadelta(model.parameters(), lr=args.lr) - - scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) - for epoch in range(1, args.epochs + 1): - train(args, model, device, train_loader, optimizer, epoch) - test(args, model, device, test_loader) - scheduler.step() - - if args.dry_run: - break - - if args.save_model: - torch.save(model.state_dict(), "mnist_cnn.pt") + hparams = parser.parse_args() + run(hparams) if __name__ == "__main__": diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py index 78677cdf33bc4..f03850b94e92c 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py @@ -26,73 +26,80 @@ from pytorch_lightning.lite import LightningLite -def train(lite, args, model, train_loader, optimizer, epoch): - model.train() - for batch_idx, (data, target) in enumerate(train_loader): - optimizer.zero_grad() - output = model(data) - loss = F.nll_loss(output, target) - lite.backward(loss) - optimizer.step() - if (batch_idx == 0) or ((batch_idx + 1) % args.log_interval == 0): - print( - "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format( - epoch, - batch_idx * len(data), - len(train_loader.dataset), - 100.0 * batch_idx / len(train_loader), - loss.item(), - ) - ) - if args.dry_run: - break - - -def test(lite, args, model, test_loader): - model.eval() - test_loss = 0 - acc = Accuracy().to(lite.device) - with torch.no_grad(): - for data, target in test_loader: - output = model(data) - test_loss += F.nll_loss(output, target, reduction="sum").item() # sum up batch loss - acc.update(output, target) - if args.dry_run: - break - - test_loss = lite.all_gather(test_loss).sum() / len(test_loader.dataset) - - if lite.is_global_zero: - print(f"\nTest set: Average loss: {test_loss:.4f}, Accuracy: ({acc.compute():.0f}%)\n") - - class Lite(LightningLite): - def run(self, args): - train_kwargs = {"batch_size": args.batch_size} - test_kwargs = {"batch_size": args.test_batch_size} + def run(self, hparams): + self.hparams = hparams + seed_everything(hparams.seed) + transform = T.Compose([T.ToTensor(), T.Normalize((0.1307,), (0.3081,))]) train_dataset = MNIST("./data", train=True, download=True, transform=transform) test_dataset = MNIST("./data", train=False, transform=transform) - train_loader = torch.utils.data.DataLoader(train_dataset, **train_kwargs) - test_loader = torch.utils.data.DataLoader(test_dataset, **test_kwargs) + train_loader = torch.utils.data.DataLoader( + train_dataset, + batch_size=hparams.batch_size, + ) + test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=hparams.batch_size) train_loader, test_loader = self.setup_dataloaders(train_loader, test_loader) model = Net() - optimizer = optim.Adadelta(model.parameters(), lr=args.lr) - + optimizer = optim.Adadelta(model.parameters(), lr=hparams.lr) model, optimizer = self.setup(model, optimizer) - scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) - for epoch in range(1, args.epochs + 1): - train(self, args, model, train_loader, optimizer, epoch) - test(self, args, model, test_loader) + scheduler = StepLR(optimizer, step_size=1, gamma=hparams.gamma) + + test_acc = Accuracy() + + # EPOCH LOOP + for epoch in range(1, hparams.epochs + 1): + + # TRAINING LOOP + model.train() + for batch_idx, (data, target) in enumerate(train_loader): + optimizer.zero_grad() + output = model(data) + loss = F.nll_loss(output, target) + + #################### + self.backward(loss) + #################### + + optimizer.step() + if (batch_idx == 0) or ((batch_idx + 1) % hparams.log_interval == 0): + print( + "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format( + epoch, + batch_idx * len(data), + len(train_loader.dataset), + 100.0 * batch_idx / len(train_loader), + loss.item(), + ) + ) + if hparams.dry_run: + break + scheduler.step() - if args.dry_run: + # TESTING LOOP + model.eval() + test_loss = 0 + with torch.no_grad(): + for data, target in test_loader: + output = model(data) + test_loss += F.nll_loss(output, target, reduction="sum").item() # sum up batch loss + test_acc(output, target) + if hparams.dry_run: + break + + test_loss = self.all_gather(test_loss).sum() / len(test_loader.dataset) + + print(f"\nTest set: Average loss: {test_loss:.4f}, Accuracy: ({test_acc.compute():.0f}%)\n") + test_acc.reset() + + if hparams.dry_run: break - if args.save_model and self.is_global_zero: + if hparams.save_model and self.is_global_zero: torch.save(model.state_dict(), "mnist_cnn.pt") @@ -102,9 +109,6 @@ def run(self, args): parser.add_argument( "--batch-size", type=int, default=64, metavar="N", help="input batch size for training (default: 64)" ) - parser.add_argument( - "--test-batch-size", type=int, default=1000, metavar="N", help="input batch size for testing (default: 1000)" - ) parser.add_argument("--epochs", type=int, default=14, metavar="N", help="number of epochs to train (default: 14)") parser.add_argument("--lr", type=float, default=1.0, metavar="LR", help="learning rate (default: 1.0)") parser.add_argument("--gamma", type=float, default=0.7, metavar="M", help="Learning rate step gamma (default: 0.7)") @@ -119,13 +123,6 @@ def run(self, args): help="how many batches to wait before logging training status", ) parser.add_argument("--save-model", action="store_true", default=False, help="For Saving the current Model") - args = parser.parse_args() - - seed_everything(args.seed) - - if torch.cuda.is_available(): - lite_kwargs = {"accelerator": "gpu", "devices": torch.cuda.device_count()} - else: - lite_kwargs = {"accelerator": "cpu"} + hparams = parser.parse_args() - Lite(**lite_kwargs).run(args) + Lite(accelerator="gpu" if torch.cuda.is_available() else "cpu", devices=torch.cuda.device_count()).run(hparams) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py index 223f23312586e..1095a6a54822f 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py @@ -26,48 +26,13 @@ from pytorch_lightning.lite import LightningLite -def train(lite, args, model, train_loader, optimizer, epoch): - model.train() - for batch_idx, batch in enumerate(train_loader): - optimizer.zero_grad() - loss = lite.training_step(batch, batch_idx) - lite.backward(loss) - optimizer.step() - if (batch_idx == 0) or ((batch_idx + 1) % args.log_interval == 0): - print( - "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format( - epoch, - batch_idx * len(batch[0]), - len(train_loader.dataset), - 100.0 * batch_idx / len(train_loader), - loss.item(), - ) - ) - if args.dry_run: - break - - -def test(lite, args, model, test_loader): - model.eval() - test_loss = 0 - with torch.no_grad(): - for batch_idx, batch in enumerate(test_loader): - test_loss += lite.test_step(batch, batch_idx) - if args.dry_run: - break - - test_loss = lite.all_gather(test_loss).sum() / len(test_loader.dataset) - - if lite.is_global_zero: - print(f"\nTest set: Average loss: {test_loss:.4f}, Accuracy: ({lite.test_acc.compute():.0f}%)\n") - - class Lite(LightningLite): """`Lite` is starting to look like a `LightningModule`.""" def run(self, hparams): self.hparams = hparams + seed_everything(hparams.seed) self.model = Net() [optimizer], [scheduler] = self.configure_optimizers() @@ -77,15 +42,49 @@ def run(self, hparams): self.prepare_data() train_loader, test_loader = self.setup_dataloaders(self.train_dataloader(), self.train_dataloader()) - self.test_acc = Accuracy() + # EPOCH LOOP for epoch in range(1, hparams.epochs + 1): - train(self, hparams, model, train_loader, optimizer, epoch) - test(self, hparams, model, test_loader) + + # TRAINING LOOP + self.model.train() + for batch_idx, batch in enumerate(train_loader): + optimizer.zero_grad() + loss = self.training_step(batch, batch_idx) + self.backward(loss) + optimizer.step() + + if (batch_idx == 0) or ((batch_idx + 1) % hparams.log_interval == 0): + print( + "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format( + epoch, + (batch_idx + 1) * self.hparams.batch_size, + len(train_loader.dataset), + 100.0 * batch_idx / len(train_loader), + loss.item(), + ) + ) + if hparams.dry_run: + break + scheduler.step() - if args.dry_run: + # TESTING LOOP + self.model.eval() + test_loss = 0 + with torch.no_grad(): + for batch_idx, batch in enumerate(test_loader): + test_loss += self.test_step(batch, batch_idx) + if hparams.dry_run: + break + + test_loss = self.all_gather(test_loss).sum() / len(test_loader.dataset) + + print(f"\nTest set: Average loss: {test_loss:.4f}, Accuracy: ({self.test_acc.compute():.0f}%)\n") + self.test_acc.reset() + + if hparams.dry_run: break if hparams.save_model and self.is_global_zero: @@ -97,12 +96,14 @@ def forward(self, x): return self.model(x) def training_step(self, batch, batch_idx): + """Here you compute and return the training loss+ compute extra training metrics.""" x, y = batch logits = self.forward(x) loss = F.nll_loss(logits, y.long()) return loss def test_step(self, batch, batch_idx): + """Here you compute and return the testing loss+ compute extra testing metrics.""" x, y = batch logits = self.forward(x) loss = F.nll_loss(logits, y.long()) @@ -137,13 +138,9 @@ def test_dataloader(self): parser.add_argument( "--batch-size", type=int, default=64, metavar="N", help="input batch size for training (default: 64)" ) - parser.add_argument( - "--test-batch-size", type=int, default=1000, metavar="N", help="input batch size for testing (default: 1000)" - ) parser.add_argument("--epochs", type=int, default=14, metavar="N", help="number of epochs to train (default: 14)") parser.add_argument("--lr", type=float, default=1.0, metavar="LR", help="learning rate (default: 1.0)") parser.add_argument("--gamma", type=float, default=0.7, metavar="M", help="Learning rate step gamma (default: 0.7)") - parser.add_argument("--no-cuda", action="store_true", default=False, help="disables CUDA training") parser.add_argument("--dry-run", action="store_true", default=False, help="quickly check a single pass") parser.add_argument("--seed", type=int, default=1, metavar="S", help="random seed (default: 1)") parser.add_argument( @@ -154,13 +151,6 @@ def test_dataloader(self): help="how many batches to wait before logging training status", ) parser.add_argument("--save-model", action="store_true", default=False, help="For Saving the current Model") - args = parser.parse_args() - - seed_everything(args.seed) - - if torch.cuda.is_available(): - lite_kwargs = {"accelerator": "gpu", "devices": torch.cuda.device_count()} - else: - lite_kwargs = {"accelerator": "cpu"} + hparams = parser.parse_args() - Lite(**lite_kwargs).run(args) + Lite(accelerator="gpu" if torch.cuda.is_available() else "cpu", devices=torch.cuda.device_count()).run(hparams) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py b/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py index 6b73bfa20bf8f..42501cdbfdddb 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py @@ -48,11 +48,9 @@ def test_step(self, batch, batch_idx): logits = self.forward(x) loss = F.nll_loss(logits, y.long()) self.test_acc(logits, y.long()) + self.log("test_acc", self.test_acc) return loss - def test_epoch_end(self, *_) -> None: - self.log("test_acc", self.test_acc.compute()) - def configure_optimizers(self): optimizer = torch.optim.Adadelta(self.model.parameters(), lr=self.hparams.lr) return [optimizer], [torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=self.hparams.gamma)] @@ -76,7 +74,8 @@ def test_dataloader(self): def cli_main(): - cli = LightningCLI(ImageClassifier, seed_everything_default=1234, save_config_overwrite=True, run=False) + # The LightningCLI removes all the boilerplate associate to arguments parsing. This is purely optional. + cli = LightningCLI(ImageClassifier, seed_everything_default=42, save_config_overwrite=True, run=False) cli.trainer.fit(cli.model, datamodule=cli.datamodule) cli.trainer.test(ckpt_path="best", datamodule=cli.datamodule) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py b/pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py index fc30836b6c37b..3dfb5543aca21 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py @@ -80,8 +80,9 @@ def test_dataloader(self): def cli_main(): + # The LightningCLI removes all the boilerplate associate to arguments parsing. This is purely optional. cli = LightningCLI( - ImageClassifier, MNISTDataModule, seed_everything_default=1234, save_config_overwrite=True, run=False + ImageClassifier, MNISTDataModule, seed_everything_default=42, save_config_overwrite=True, run=False ) cli.trainer.fit(cli.model, datamodule=cli.datamodule) cli.trainer.test(ckpt_path="best", datamodule=cli.datamodule) diff --git a/pytorch_lightning/callbacks/quantization.py b/pytorch_lightning/callbacks/quantization.py index bf0088575e8b4..ca82a574f71d1 100644 --- a/pytorch_lightning/callbacks/quantization.py +++ b/pytorch_lightning/callbacks/quantization.py @@ -33,10 +33,10 @@ import pytorch_lightning as pl from pytorch_lightning.callbacks.base import Callback -from pytorch_lightning.utilities import _TORCH_GREATER_EQUAL_DEV_1_10 +from pytorch_lightning.utilities import _TORCH_GREATER_EQUAL_1_10 from pytorch_lightning.utilities.exceptions import MisconfigurationException -if _TORCH_GREATER_EQUAL_DEV_1_10: +if _TORCH_GREATER_EQUAL_1_10: from torch.ao.quantization.qconfig import QConfig else: from torch.quantization import QConfig @@ -245,7 +245,7 @@ def on_fit_start(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule") - # version=None corresponds to using FakeQuantize rather than # FusedMovingAvgObsFakeQuantize which was introduced in PT1.10 # details in https://github.com/pytorch/pytorch/issues/64564 - extra_kwargs = dict(version=None) if _TORCH_GREATER_EQUAL_DEV_1_10 else {} + extra_kwargs = dict(version=None) if _TORCH_GREATER_EQUAL_1_10 else {} pl_module.qconfig = torch.quantization.get_default_qat_qconfig(self._qconfig, **extra_kwargs) elif isinstance(self._qconfig, QConfig): diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 7a58f91adda7d..cfac84be1367b 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -39,7 +39,7 @@ from pytorch_lightning.trainer.connectors.logger_connector.fx_validator import _FxValidator from pytorch_lightning.utilities import ( _IS_WINDOWS, - _TORCH_GREATER_EQUAL_DEV_1_10, + _TORCH_GREATER_EQUAL_1_10, GradClipAlgorithmType, rank_zero_deprecation, rank_zero_warn, @@ -2043,7 +2043,7 @@ def _register_sharded_tensor_state_dict_hooks_if_available(self) -> None: These hooks ensure that ShardedTensors are included when saving, and are loaded the LightningModule correctly. """ - if not _TORCH_GREATER_EQUAL_DEV_1_10 or _IS_WINDOWS: + if not _TORCH_GREATER_EQUAL_1_10 or _IS_WINDOWS: return from torch.distributed._sharded_tensor import pre_load_state_dict_hook, state_dict_hook diff --git a/pytorch_lightning/plugins/precision/native_amp.py b/pytorch_lightning/plugins/precision/native_amp.py index 3fc903cbb3fce..487d80005c222 100644 --- a/pytorch_lightning/plugins/precision/native_amp.py +++ b/pytorch_lightning/plugins/precision/native_amp.py @@ -21,10 +21,10 @@ import pytorch_lightning as pl from pytorch_lightning.plugins.precision.mixed import MixedPrecisionPlugin -from pytorch_lightning.utilities import _TORCH_GREATER_EQUAL_DEV_1_10, AMPType +from pytorch_lightning.utilities import _TORCH_GREATER_EQUAL_1_10, AMPType from pytorch_lightning.utilities.exceptions import MisconfigurationException -if _TORCH_GREATER_EQUAL_DEV_1_10: +if _TORCH_GREATER_EQUAL_1_10: from torch import autocast else: from torch.cuda.amp import autocast @@ -47,7 +47,7 @@ def __init__(self, precision: Union[int, str] = 16, use_cpu: bool = False) -> No def _select_precision_dtype(self, precision: Union[int, str] = 16) -> torch.dtype: if precision == "bf16": - if not _TORCH_GREATER_EQUAL_DEV_1_10: + if not _TORCH_GREATER_EQUAL_1_10: raise MisconfigurationException( "To use bfloat16 with native amp you must install torch greater or equal to 1.10." ) @@ -97,7 +97,7 @@ def optimizer_step( self.scaler.update() def autocast_context_manager(self) -> autocast: - if _TORCH_GREATER_EQUAL_DEV_1_10: + if _TORCH_GREATER_EQUAL_1_10: return autocast("cpu" if self.use_cpu else "cuda", dtype=self._dtype) return autocast() diff --git a/pytorch_lightning/utilities/__init__.py b/pytorch_lightning/utilities/__init__.py index bc19aa1366a55..158d7356c91ce 100644 --- a/pytorch_lightning/utilities/__init__.py +++ b/pytorch_lightning/utilities/__init__.py @@ -48,7 +48,6 @@ _TORCH_GREATER_EQUAL_1_8, _TORCH_GREATER_EQUAL_1_9, _TORCH_GREATER_EQUAL_1_10, - _TORCH_GREATER_EQUAL_DEV_1_10, _TORCH_QUANTIZE_AVAILABLE, _TORCHTEXT_AVAILABLE, _TORCHVISION_AVAILABLE, diff --git a/pytorch_lightning/utilities/imports.py b/pytorch_lightning/utilities/imports.py index c7ad70895672a..811e81a370601 100644 --- a/pytorch_lightning/utilities/imports.py +++ b/pytorch_lightning/utilities/imports.py @@ -75,7 +75,7 @@ def _compare_version(package: str, op: Callable, version: str, use_base_version: _TORCH_GREATER_EQUAL_1_8_1 = _compare_version("torch", operator.ge, "1.8.1") _TORCH_GREATER_EQUAL_1_9 = _compare_version("torch", operator.ge, "1.9.0") _TORCH_GREATER_EQUAL_1_10 = _compare_version("torch", operator.ge, "1.10.0") -_TORCH_GREATER_EQUAL_DEV_1_10 = _compare_version("torch", operator.ge, "1.10.0", use_base_version=True) +# _TORCH_GREATER_EQUAL_DEV_1_11 = _compare_version("torch", operator.ge, "1.11.0", use_base_version=True) _APEX_AVAILABLE = _module_available("apex.amp") _DEEPSPEED_AVAILABLE = _module_available("deepspeed") diff --git a/tests/core/test_lightning_module.py b/tests/core/test_lightning_module.py index d661228ee09d8..ff8ffa3c50acd 100644 --- a/tests/core/test_lightning_module.py +++ b/tests/core/test_lightning_module.py @@ -21,7 +21,6 @@ from pytorch_lightning import Trainer from pytorch_lightning.loggers import TensorBoardLogger -from pytorch_lightning.utilities import _IS_WINDOWS, _TORCH_GREATER_EQUAL_DEV_1_10 from pytorch_lightning.utilities.exceptions import MisconfigurationException from tests.helpers import BoringModel from tests.helpers.runif import RunIf @@ -312,10 +311,7 @@ def __init__(self, spec): self.sharded_tensor.local_shards()[0].tensor.fill_(0) -@pytest.mark.skipif( - not _TORCH_GREATER_EQUAL_DEV_1_10, reason="Test requires the torch version to support `ShardedTensor`" -) -@pytest.mark.skipif(_IS_WINDOWS, reason="Not supported on Windows") +@RunIf(min_torch="1.10", skip_windows=True) def test_sharded_tensor_state_dict(tmpdir, single_process_pg): spec = dist._sharding_spec.ChunkShardingSpec( dim=0, diff --git a/tests/models/test_amp.py b/tests/models/test_amp.py index 716c0f17f203d..86863238da057 100644 --- a/tests/models/test_amp.py +++ b/tests/models/test_amp.py @@ -22,7 +22,6 @@ import tests.helpers.utils as tutils from pytorch_lightning import Trainer from pytorch_lightning.plugins.environments import SLURMEnvironment -from pytorch_lightning.utilities.imports import _TORCH_GREATER_EQUAL_DEV_1_10 from tests.helpers import BoringModel, RandomDataset from tests.helpers.runif import RunIf @@ -68,7 +67,7 @@ def _assert_autocast_enabled(self): assert torch.is_autocast_enabled() -@pytest.mark.skipif(not _TORCH_GREATER_EQUAL_DEV_1_10, reason="Needs bfloat16 support") +@RunIf(min_torch="1.10") @pytest.mark.parametrize( "strategy", [ @@ -95,8 +94,7 @@ def test_amp_cpus(tmpdir, strategy, precision, num_processes): assert trainer.state.finished, f"Training failed with {trainer.state}" -@RunIf(min_gpus=2) -@pytest.mark.skipif(not _TORCH_GREATER_EQUAL_DEV_1_10, reason="Needs bfloat16 support") +@RunIf(min_gpus=2, min_torch="1.10") @pytest.mark.parametrize("strategy", [None, "dp", "ddp_spawn"]) @pytest.mark.parametrize("precision", [16, "bf16"]) @pytest.mark.parametrize("gpus", [1, 2]) diff --git a/tests/plugins/test_amp_plugins.py b/tests/plugins/test_amp_plugins.py index ed8c653b3a78f..227d898a7da40 100644 --- a/tests/plugins/test_amp_plugins.py +++ b/tests/plugins/test_amp_plugins.py @@ -21,7 +21,6 @@ from pytorch_lightning import Trainer from pytorch_lightning.plugins import ApexMixedPrecisionPlugin, NativeMixedPrecisionPlugin from pytorch_lightning.plugins.precision import MixedPrecisionPlugin -from pytorch_lightning.utilities import _TORCH_GREATER_EQUAL_DEV_1_10 from pytorch_lightning.utilities.exceptions import MisconfigurationException from tests.helpers import BoringModel from tests.helpers.runif import RunIf @@ -178,7 +177,7 @@ def test_amp_apex_ddp_spawn_fit(amp_level, tmpdir): trainer.fit(model) -@pytest.mark.skipif(not _TORCH_GREATER_EQUAL_DEV_1_10, reason="Torch CPU AMP is not available.") +@RunIf(min_torch="1.10") def test_cpu_amp_precision_context_manager(tmpdir): """Test to ensure that the context manager correctly is set to CPU + bfloat16, and a scaler isn't set.""" plugin = NativeMixedPrecisionPlugin(precision="bf16", use_cpu=True) @@ -197,7 +196,7 @@ def test_precision_selection_raises(monkeypatch): import pytorch_lightning.plugins.precision.native_amp as amp - monkeypatch.setattr(amp, "_TORCH_GREATER_EQUAL_DEV_1_10", False) + monkeypatch.setattr(amp, "_TORCH_GREATER_EQUAL_1_10", False) with pytest.warns( UserWarning, match=r"precision=16\)` but native AMP is not supported on CPU. Using `precision='bf16" ), pytest.raises(MisconfigurationException, match="must install torch greater or equal to 1.10"): From 93b7940b10575a6c90806f5ed380daac4f61a7b2 Mon Sep 17 00:00:00 2001 From: tchaton Date: Thu, 28 Oct 2021 09:28:01 +0100 Subject: [PATCH 247/331] update --- tests/helpers/pipelines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/pipelines.py b/tests/helpers/pipelines.py index 3e5066d708da0..643d3e50cb894 100644 --- a/tests/helpers/pipelines.py +++ b/tests/helpers/pipelines.py @@ -67,7 +67,7 @@ def run_model_test( assert trainer.state.finished, f"Training failed with {trainer.state}" # Check that the model is actually changed post-training change_ratio = torch.norm(initial_values - post_train_values) - assert change_ratio > 0.1, f"the model is changed of {change_ratio}" + assert change_ratio > 0.03, f"the model is changed of {change_ratio}" # test model loading pretrained_model = load_model_from_checkpoint(logger, trainer.checkpoint_callback.best_model_path, type(model)) From a6414a2f25cbe6bbc4fed6b87225ff4fee0ee7c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 28 Oct 2021 11:41:53 +0200 Subject: [PATCH 248/331] update access to deepspeed internal vars --- pytorch_lightning/lite/lite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 075b3ad729b36..e373ce3bfa3a7 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -365,7 +365,7 @@ def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> Any: run_method = partial(self._run_with_sharded_context, run_method) if isinstance(self._strategy, DDPSpawnPlugin): - return self._strategy.spawn(run_method, *args, **kwargs) + return self._strategy.spawn(run_method, *args, return_result=True, **kwargs) else: return run_method(*args, **kwargs) @@ -402,7 +402,7 @@ def _set_deepspeed_precision_variables(self) -> None: amp_type = self._accelerator_connector.amp_type amp_level = self._accelerator_connector.amp_level precision = self._accelerator_connector.precision - self._strategy.amp_level, self._strategy.amp_type, self._strategy._precision = amp_level, amp_type, precision + self._strategy._amp_level, self._strategy._amp_type, self._strategy._precision = amp_level, amp_type, precision def _requires_distributed_sampler(self, dataloader: DataLoader) -> bool: return ( From 5e1aeb89569484c76b6dec7a1cdcbc38ad04b8ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 28 Oct 2021 12:38:31 +0200 Subject: [PATCH 249/331] fix check for multiple models in deepspeed --- pytorch_lightning/lite/lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index e373ce3bfa3a7..1471a13c2e70d 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -255,7 +255,7 @@ def backward(self, tensor: Tensor, *args: Any, model: Optional[_LiteModule] = No model as argument here. """ module = model.module if model is not None else model - if self._num_models > 0 and isinstance(self._strategy, DeepSpeedPlugin): + if self._num_models > 1 and isinstance(self._strategy, DeepSpeedPlugin): if model is None: raise MisconfigurationException( "When using multiple models + deepspeed, please provide the model used to perform the optimization." From f885b35c7e267e2b43007a70903eb73b104e2166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 28 Oct 2021 13:32:52 +0200 Subject: [PATCH 250/331] fix deepspeed precision --- pytorch_lightning/lite/lite.py | 20 ++++++++++++------- .../plugins/training_type/deepspeed.py | 3 ++- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 1471a13c2e70d..71779670121ed 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -255,14 +255,20 @@ def backward(self, tensor: Tensor, *args: Any, model: Optional[_LiteModule] = No model as argument here. """ module = model.module if model is not None else model - if self._num_models > 1 and isinstance(self._strategy, DeepSpeedPlugin): + if isinstance(self._strategy, DeepSpeedPlugin): if model is None: - raise MisconfigurationException( - "When using multiple models + deepspeed, please provide the model used to perform the optimization." - ) - - # requires to attach the current `DeepSpeedEngine` for the `_LiteOptimizer.step` call. - self._strategy.model = module + if self._num_models == 0: + raise MisconfigurationException( + "No models were setup for backward. Did you forget to call `self.setup`?" + ) + if self._num_models > 1: + raise MisconfigurationException( + "When using multiple models + deepspeed, please provide the model used to perform the optimization." + ) + module = self._strategy.model + else: + # requires to attach the current `DeepSpeedEngine` for the `_LiteOptimizer.step` call. + self._strategy.model = module self._precision_plugin._run_backward(tensor, module, *args, **kwargs) diff --git a/pytorch_lightning/plugins/training_type/deepspeed.py b/pytorch_lightning/plugins/training_type/deepspeed.py index f6b5481dd5ef9..b06406570306c 100644 --- a/pytorch_lightning/plugins/training_type/deepspeed.py +++ b/pytorch_lightning/plugins/training_type/deepspeed.py @@ -336,7 +336,8 @@ def precision(self) -> Union[str, int]: @property def amp_level(self) -> Optional[str]: - return self._amp_level or self.lightning_module.trainer._accelerator_connector.amp_level + if self._amp_type == AMPType.APEX: + return self._amp_level or self.lightning_module.trainer._accelerator_connector.amp_level @property def amp_type(self) -> Optional[str]: From de4ef790c0f8a0a81924a17f26d287fa92d3c8bd Mon Sep 17 00:00:00 2001 From: tchaton Date: Thu, 28 Oct 2021 13:45:58 +0100 Subject: [PATCH 251/331] update --- .../mnist_examples/image_classifier_2_lite.py | 28 ++++++++++++++----- .../image_classifier_3_lite_to_lightning.py | 2 +- pytorch_lightning/lite/lite.py | 8 +++++- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py index f03850b94e92c..bdf55958b6861 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py @@ -40,14 +40,19 @@ def run(self, hparams): ) test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=hparams.batch_size) + # don't forget to call `setup_dataloaders` to prepare for dataloaders for distributed training. train_loader, test_loader = self.setup_dataloaders(train_loader, test_loader) - model = Net() + model = Net() # remove call to .to(device) optimizer = optim.Adadelta(model.parameters(), lr=hparams.lr) + + # don't forget to call `setup` to prepare for model / optimizer for distributed training. + # the model is moved automatically to the right device. model, optimizer = self.setup(model, optimizer) scheduler = StepLR(optimizer, step_size=1, gamma=hparams.gamma) + # use torchmetrics instead of manually computing the accuracy test_acc = Accuracy() # EPOCH LOOP @@ -56,13 +61,11 @@ def run(self, hparams): # TRAINING LOOP model.train() for batch_idx, (data, target) in enumerate(train_loader): + # NOTE: no need to call .to(device) on the data, target" optimizer.zero_grad() output = model(data) loss = F.nll_loss(output, target) - - #################### - self.backward(loss) - #################### + self.backward(loss) # instead of loss.backward() optimizer.step() if (batch_idx == 0) or ((batch_idx + 1) % hparams.log_interval == 0): @@ -85,12 +88,21 @@ def run(self, hparams): test_loss = 0 with torch.no_grad(): for data, target in test_loader: + # NOTE: no need to call .to(device) on the data, target" output = model(data) - test_loss += F.nll_loss(output, target, reduction="sum").item() # sum up batch loss + test_loss += F.nll_loss(output, target, reduction="sum").item() + + # WITHOUT TorchMetrics + # pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability + # correct += pred.eq(target.view_as(pred)).sum().item() + + # WITH TorchMetrics test_acc(output, target) + if hparams.dry_run: break + # all_gather is used to aggregated the value across processes test_loss = self.all_gather(test_loss).sum() / len(test_loader.dataset) print(f"\nTest set: Average loss: {test_loss:.4f}, Accuracy: ({test_acc.compute():.0f}%)\n") @@ -99,7 +111,9 @@ def run(self, hparams): if hparams.dry_run: break - if hparams.save_model and self.is_global_zero: + # When using distributed training, use `self.can_save_checkpoint` + # to ensure the current process is allowed to save a checkpoint + if hparams.save_model and self.can_save_checkpoint: torch.save(model.state_dict(), "mnist_cnn.pt") diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py index 1095a6a54822f..ae38f6a1c8e78 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py @@ -87,7 +87,7 @@ def run(self, hparams): if hparams.dry_run: break - if hparams.save_model and self.is_global_zero: + if hparams.save_model and self.can_save_checkpoint: torch.save(model.state_dict(), "mnist_cnn.pt") # Functions for the `LightningModule` conversion diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 246632cb1aa47..e21a65c5fb10a 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -16,7 +16,7 @@ from contextlib import contextmanager from functools import partial from pathlib import Path -from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, overload, Sequence, Tuple, Union +from typing import Any, Callable, Dict, Generator, Iterable, List, Optional, Sequence, Tuple, Union import torch import torch.nn as nn @@ -140,6 +140,12 @@ def is_global_zero(self) -> bool: """Wether this rank is rank zero.""" return self._strategy.is_global_zero + @property + def can_save_checkpoint(self) -> bool: + if isinstance(self._strategy, DeepSpeedPlugin): + return True + return self.is_global_zero + @abstractmethod def run(self, *args: Any, **kwargs: Any) -> Any: """All the code inside this run method gets accelerated by Lite. From 7f623940f2f95d69046b1a1da37499c531f06cce Mon Sep 17 00:00:00 2001 From: tchaton Date: Thu, 28 Oct 2021 13:46:22 +0100 Subject: [PATCH 252/331] update --- tests/lite/test_parity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/lite/test_parity.py b/tests/lite/test_parity.py index 4b52448ceff71..48ed2bb22cd98 100644 --- a/tests/lite/test_parity.py +++ b/tests/lite/test_parity.py @@ -34,7 +34,7 @@ from pytorch_lightning.plugins.training_type.ddp_spawn import DDPSpawnPlugin from pytorch_lightning.utilities.apply_func import apply_to_collection, move_data_to_device from pytorch_lightning.utilities.cloud_io import atomic_save -from pytorch_lightning.utilities.imports import _TORCH_GREATER_EQUAL_DEV_1_10 +from pytorch_lightning.utilities.imports import _TORCH_GREATER_EQUAL_1_10 from tests.helpers.boring_model import RandomDataset from tests.helpers.runif import RunIf @@ -123,7 +123,7 @@ def precision_context(precision, accelerator) -> Generator[None, None, None]: 1, "gpu", marks=pytest.mark.skipif( - not (_TORCH_GREATER_EQUAL_DEV_1_10 and is_available()), + not (_TORCH_GREATER_EQUAL_1_10 and is_available()), reason="bfloat16 and requires GPU isn't available.", ), ), From db34e0907e433b9d798679382119f75e2a355100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 28 Oct 2021 15:25:07 +0200 Subject: [PATCH 253/331] fix line too long --- pytorch_lightning/lite/lite.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 71779670121ed..1862143307663 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -263,7 +263,8 @@ def backward(self, tensor: Tensor, *args: Any, model: Optional[_LiteModule] = No ) if self._num_models > 1: raise MisconfigurationException( - "When using multiple models + deepspeed, please provide the model used to perform the optimization." + "When using multiple models + deepspeed, please provide the model used to perform" + " the optimization." ) module = self._strategy.model else: From 992fd45661631b65986028e1fe17ec627a1a3564 Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Thu, 28 Oct 2021 15:57:37 +0200 Subject: [PATCH 254/331] Minor changes --- pytorch_lightning/lite/lite.py | 20 ++++++++++---------- tests/lite/test_lite.py | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 1862143307663..7d751adcc139b 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -103,7 +103,7 @@ def __init__( self._accelerator = self._accelerator_connector.accelerator self._strategy = self._accelerator.training_type_plugin self._precision_plugin = self._accelerator.precision_plugin - self._num_models: int = 0 + self._models_setup: int = 0 # wrap the run method so we can inject setup logic or spawn processes for the user setattr(self, "run", self._run_wrapper(self.run)) @@ -146,8 +146,8 @@ def run(self, *args: Any, **kwargs: Any) -> Any: """All the code inside this run method gets accelerated by Lite. Args: - *args: Add any positional arguments you need, e.g., the hyperparameters for your model - **kwargs: Add any keyword arguments you need, e.g., the hyperparameters for your model + *args: Add any positional arguments you need, e.g., the hyperparameters for your model. + **kwargs: Add any keyword arguments you need, e.g., the hyperparameters for your model. """ def setup( @@ -176,7 +176,7 @@ def setup( model, optimizers = self._strategy._setup_model_and_optimizers(model, list(optimizers)) model = _LiteModule(model, self._accelerator) optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self._accelerator) for optimizer in optimizers] - self._num_models += 1 + self._models_setup += 1 if optimizers: return [model] + optimizers # type: ignore return model @@ -251,20 +251,20 @@ def backward(self, tensor: Tensor, *args: Any, model: Optional[_LiteModule] = No **kwargs: Optional named keyword arguments passed to the underlying backward function. Note: - When using ``strategy='deepspeed"`` and multiple models were setup, it is required to pass in the + When using ``strategy="deepspeed"`` and multiple models were setup, it is required to pass in the model as argument here. """ module = model.module if model is not None else model if isinstance(self._strategy, DeepSpeedPlugin): if model is None: - if self._num_models == 0: + if self._models_setup == 0: raise MisconfigurationException( - "No models were setup for backward. Did you forget to call `self.setup`?" + "No models were setup for backward. Did you forget to call `self.setup()`?" ) - if self._num_models > 1: + if self._models_setup > 1: raise MisconfigurationException( "When using multiple models + deepspeed, please provide the model used to perform" - " the optimization." + " the optimization: `self.backward(loss, model=model)`" ) module = self._strategy.model else: @@ -450,7 +450,7 @@ def _supported_device_types() -> Sequence[DeviceType]: ) @staticmethod - def _supported_strategy_types() -> Sequence[str]: + def _supported_strategy_types() -> Sequence[DistributedType]: return ( DistributedType.DP, DistributedType.DDP, diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index f47f9f1df1434..8ff8ccf863649 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -129,12 +129,12 @@ def test_setup_tracks_num_models(): model = nn.Linear(1, 2) optimizer = torch.optim.Adam(model.parameters()) - assert lite._num_models == 0 + assert lite._models_setup == 0 lite.setup(model, optimizer) - assert lite._num_models == 1 + assert lite._models_setup == 1 lite.setup(model, optimizer) - assert lite._num_models == 2 + assert lite._models_setup == 2 def test_setup_dataloaders_unsupported_type(): From b8d44ce09085775c9165c5a69c7b6c67e28403f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 28 Oct 2021 16:11:55 +0200 Subject: [PATCH 255/331] remove identity wrapper --- pytorch_lightning/lite/lite.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 1862143307663..d83b2bd6cf165 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -106,7 +106,7 @@ def __init__( self._num_models: int = 0 # wrap the run method so we can inject setup logic or spawn processes for the user - setattr(self, "run", self._run_wrapper(self.run)) + setattr(self, "run", partial(self._run_impl, self.run)) @property def device(self) -> torch.device: @@ -361,9 +361,6 @@ def save_checkpoint(self, filepath: Union[str, Path], content: Dict[str, Any]) - """ self._strategy.save_checkpoint(content, filepath) - def _run_wrapper(self, run_method: Callable) -> Callable: - return partial(self._run_impl, run_method) - def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> Any: self._set_plugin_specific_precision_variables() self._accelerator.setup_environment() From 04094c39cd69567407603ac54d0952b6d03f06c4 Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Thu, 28 Oct 2021 16:15:55 +0200 Subject: [PATCH 256/331] Same annotations as Lightning which are identical to those in torch --- pytorch_lightning/lite/wrappers.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index e1d16ca8a3384..9c6641bfb4413 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Callable, Dict, Generator, Iterator, List, Optional, Union +from typing import Any, Callable, Generator, Iterator, List, Optional, Union import torch from torch import nn as nn @@ -48,27 +48,27 @@ def optimizer(self) -> Optimizer: return self._optimizer @property - def state(self) -> Dict[str, torch.Tensor]: - return self._optimizer.state - - @state.setter - def state(self, state: Dict[str, torch.Tensor]) -> None: - self._optimizer.state = state - - @property - def defaults(self) -> Dict[str, Any]: + def defaults(self) -> dict: return self._optimizer.defaults @defaults.setter - def defaults(self, defaults: Dict[str, Any]) -> None: + def defaults(self, defaults: dict) -> None: self._optimizer.defaults = defaults @property - def param_groups(self) -> List[Dict[str, torch.Tensor]]: + def state(self) -> dict: + return self._optimizer.state + + @state.setter + def state(self, state: dict) -> None: + self._optimizer.state = state + + @property + def param_groups(self) -> List[dict]: return self._optimizer.param_groups @param_groups.setter - def param_groups(self, param_groups: List[Dict[str, torch.Tensor]]) -> None: + def param_groups(self, param_groups: List[dict]) -> None: self._optimizer.param_groups = param_groups def step(self, closure: Optional[Callable] = None) -> None: From 1d9920ac090698ad4eebc1984b08a50822b48eca Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Thu, 28 Oct 2021 16:17:21 +0200 Subject: [PATCH 257/331] Add comment --- pytorch_lightning/lite/lite.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index e9770531f66ed..0b749fc85c861 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -391,6 +391,7 @@ def _move_model_to_device(self, model: nn.Module, optimizers: List[Optimizer]) - # remains invalid. We need to update the references to point to the parameter tensors on the device. params_on_cpu = dict(model.named_parameters()) model = self.to_device(model) + # XLA makes a copy on the parameters, so the device should is not the same before and after to_device. params_on_device = dict(model.named_parameters()) mapping = {param: params_on_device[name] for name, param in params_on_cpu.items()} From a6df052f902312d3c43bba41129b3543a9837556 Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Thu, 28 Oct 2021 16:31:15 +0200 Subject: [PATCH 258/331] Simplify _LiteOptimizer --- pytorch_lightning/lite/wrappers.py | 39 +++--------------------------- 1 file changed, 4 insertions(+), 35 deletions(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 9c6641bfb4413..c1612e7298ab3 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Callable, Generator, Iterator, List, Optional, Union +from typing import Any, Callable, Generator, Iterator, Optional, Union import torch from torch import nn as nn @@ -38,51 +38,20 @@ def __init__(self, optimizer: Optimizer, accelerator: Accelerator) -> None: optimizer: The optimizer to wrap accelerator: Reference to the accelerator for handling the optimizer step """ - self.__dict__ = {k: v for k, v in optimizer.__dict__.items() if k not in ("step", "__del__")} + self.__dict__ = {k: v for k, v in optimizer.__dict__.items() if k not in ("step",)} self.__class__ = type("Lite" + optimizer.__class__.__name__, (self.__class__, optimizer.__class__), {}) - self._optimizer = optimizer + self.optimizer = optimizer self._accelerator = accelerator - @property - def optimizer(self) -> Optimizer: - return self._optimizer - - @property - def defaults(self) -> dict: - return self._optimizer.defaults - - @defaults.setter - def defaults(self, defaults: dict) -> None: - self._optimizer.defaults = defaults - - @property - def state(self) -> dict: - return self._optimizer.state - - @state.setter - def state(self, state: dict) -> None: - self._optimizer.state = state - - @property - def param_groups(self) -> List[dict]: - return self._optimizer.param_groups - - @param_groups.setter - def param_groups(self, param_groups: List[dict]) -> None: - self._optimizer.param_groups = param_groups - def step(self, closure: Optional[Callable] = None) -> None: closure = closure or _do_nothing_closure self._accelerator.optimizer_step( - self._optimizer, + self.optimizer, opt_idx=0, lambda_closure=closure, model=self._accelerator.model, ) - def zero_grad(self, *args: Any, **kwargs: Any) -> None: - self._optimizer.zero_grad(*args, **kwargs) - class _LiteModule(nn.Module): # TODO: Pass in the precision plugin instead of accelerator From 5208e19544005c34a603da8f9a30f9c9861302bc Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Thu, 28 Oct 2021 16:44:29 +0200 Subject: [PATCH 259/331] Didn't mean to remove this :) --- pytorch_lightning/lite/wrappers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index c1612e7298ab3..fe42778dcb39e 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -40,9 +40,13 @@ def __init__(self, optimizer: Optimizer, accelerator: Accelerator) -> None: """ self.__dict__ = {k: v for k, v in optimizer.__dict__.items() if k not in ("step",)} self.__class__ = type("Lite" + optimizer.__class__.__name__, (self.__class__, optimizer.__class__), {}) - self.optimizer = optimizer + self._optimizer = optimizer self._accelerator = accelerator + @property + def optimizer(self) -> Optimizer: + return self._optimizer + def step(self, closure: Optional[Callable] = None) -> None: closure = closure or _do_nothing_closure self._accelerator.optimizer_step( From 31406aeaef9211f3478a4b62c29741396a4aa5ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 28 Oct 2021 16:58:22 +0200 Subject: [PATCH 260/331] rename cast to autocast --- pytorch_lightning/lite/lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 0b749fc85c861..42eb49f7a9e44 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -274,7 +274,7 @@ def backward(self, tensor: Tensor, *args: Any, model: Optional[_LiteModule] = No self._precision_plugin._run_backward(tensor, module, *args, **kwargs) @contextmanager - def cast(self) -> Generator[None, None, None]: + def autocast(self) -> Generator[None, None, None]: """A context manager to automatically convert operations for the chosen precision. Use this only if the `forward` method of your model does not cover all operations you wish to run with the From bda0f8ab6c92ac8a09c6f9556185c980435d5710 Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Thu, 28 Oct 2021 17:00:21 +0200 Subject: [PATCH 261/331] test: Remove unused parametrization --- tests/lite/test_lite.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 8ff8ccf863649..6135d67d0d026 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -47,14 +47,14 @@ def forward(self, x): return torch.nn.functional.mse_loss(x, torch.ones_like(x)) -@pytest.mark.parametrize("accelerator", ["coconut"]) -def test_unsupported_accelerator(accelerator): +def test_unsupported_accelerator(): + accelerator = "coconut" with pytest.raises(MisconfigurationException, match=f"`accelerator={repr(accelerator)}` is not a valid choice"): EmptyLite(accelerator=accelerator) -@pytest.mark.parametrize("strategy", ["coconut"]) -def test_unsupported_strategy(strategy): +def test_unsupported_strategy(): + strategy = "coconut" with pytest.raises(MisconfigurationException, match=f"`strategy={repr(strategy)}` is not a valid choice"): EmptyLite(strategy=strategy) From c34d00606de8f71dea1a00ca9efe63919b534985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 28 Oct 2021 17:01:20 +0200 Subject: [PATCH 262/331] rename save_checkpoint to save --- pytorch_lightning/lite/lite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 42eb49f7a9e44..a54e59c7d5017 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -349,15 +349,15 @@ def all_gather( def broadcast(self, obj: object, src: int = 0) -> object: return self._strategy.broadcast(obj, src=src) - def save_checkpoint(self, filepath: Union[str, Path], content: Dict[str, Any]) -> None: + def save(self, content: Dict[str, Any], filepath: Union[str, Path]) -> None: """Save a checkpoint contents to a file. How and which processes save gets determined by the `strategy`. For example, the `ddp` strategy saves checkpoints only on process 0. Args: - filepath: A path to where the file should be saved content: A dictionary with contents, i.e., the state dict of your model + filepath: A path to where the file should be saved """ self._strategy.save_checkpoint(content, filepath) From f45c2c8223633590f6ffd3ac56689415ed67b020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 28 Oct 2021 17:01:58 +0200 Subject: [PATCH 263/331] update docstring --- pytorch_lightning/lite/lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index a54e59c7d5017..3230f63db89cb 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -350,7 +350,7 @@ def broadcast(self, obj: object, src: int = 0) -> object: return self._strategy.broadcast(obj, src=src) def save(self, content: Dict[str, Any], filepath: Union[str, Path]) -> None: - """Save a checkpoint contents to a file. + """Save checkpoint contents to a file. How and which processes save gets determined by the `strategy`. For example, the `ddp` strategy saves checkpoints only on process 0. From c84acb1743016303a206957c640aceff016d15e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 28 Oct 2021 17:02:41 +0200 Subject: [PATCH 264/331] update comment --- pytorch_lightning/lite/lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 3230f63db89cb..2c1bf2a8ce393 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -391,7 +391,7 @@ def _move_model_to_device(self, model: nn.Module, optimizers: List[Optimizer]) - # remains invalid. We need to update the references to point to the parameter tensors on the device. params_on_cpu = dict(model.named_parameters()) model = self.to_device(model) - # XLA makes a copy on the parameters, so the device should is not the same before and after to_device. + # XLA makes a copy on the parameters, so the device is not the same before and after to_device. params_on_device = dict(model.named_parameters()) mapping = {param: params_on_device[name] for name, param in params_on_cpu.items()} From 92752e66e2cc988ebc2e30a5dd5ce018ff43012d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 28 Oct 2021 17:05:51 +0200 Subject: [PATCH 265/331] add load --- pytorch_lightning/lite/lite.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 2c1bf2a8ce393..d913ea580f912 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -361,6 +361,16 @@ def save(self, content: Dict[str, Any], filepath: Union[str, Path]) -> None: """ self._strategy.save_checkpoint(content, filepath) + def load(self, filepath: Union[str, Path]) -> Any: + """Load a checkpoint from a file. + + How and which processes load gets determined by the `strategy` + + Args: + filepath: A path to where the file is located + """ + return self._strategy.load_checkpoint(filepath) + def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> Any: self._set_plugin_specific_precision_variables() self._accelerator.setup_environment() From c0ffc712d67d5f1afd9033b69a3dea376efa1885 Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Thu, 28 Oct 2021 17:11:22 +0200 Subject: [PATCH 266/331] tests: update autocast use --- tests/lite/test_parity.py | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/tests/lite/test_parity.py b/tests/lite/test_parity.py index 48ed2bb22cd98..2c7c58d249a13 100644 --- a/tests/lite/test_parity.py +++ b/tests/lite/test_parity.py @@ -23,7 +23,6 @@ import torch.multiprocessing as mp import torch.nn.functional from torch import nn -from torch.cuda import is_available from torch.nn.parallel.distributed import DistributedDataParallel from torch.utils.data import DataLoader from torch.utils.data.distributed import DistributedSampler @@ -34,7 +33,6 @@ from pytorch_lightning.plugins.training_type.ddp_spawn import DDPSpawnPlugin from pytorch_lightning.utilities.apply_func import apply_to_collection, move_data_to_device from pytorch_lightning.utilities.cloud_io import atomic_save -from pytorch_lightning.utilities.imports import _TORCH_GREATER_EQUAL_1_10 from tests.helpers.boring_model import RandomDataset from tests.helpers.runif import RunIf @@ -100,14 +98,11 @@ def precision_context(precision, accelerator) -> Generator[None, None, None]: if precision == 32: yield return - if precision == 16 and accelerator == "gpu": + if accelerator == "gpu": with torch.cuda.amp.autocast(): yield elif accelerator == "cpu": - with torch.cpu.amp.autocast(dtype=torch.float16 if precision == 16 else torch.bfloat16): - yield - else: - with torch.cuda.amp.autocast(): + with torch.cpu.amp.autocast(): yield @@ -115,18 +110,9 @@ def precision_context(precision, accelerator) -> Generator[None, None, None]: "precision, strategy, devices, accelerator", [ pytest.param(32, None, 1, "cpu"), - pytest.param(32, None, 1, "gpu", marks=pytest.mark.skipif(not is_available(), reason="requires a GPU")), - pytest.param(16, None, 1, "gpu", marks=pytest.mark.skipif(not is_available(), reason="requires a GPU")), - pytest.param( - "bf16", - None, - 1, - "gpu", - marks=pytest.mark.skipif( - not (_TORCH_GREATER_EQUAL_1_10 and is_available()), - reason="bfloat16 and requires GPU isn't available.", - ), - ), + pytest.param(32, None, 1, "gpu", marks=RunIf(min_gpus=1)), + pytest.param(16, None, 1, "gpu", marks=RunIf(min_gpus=1)), + pytest.param("bf16", None, 1, "gpu", marks=RunIf(min_torch="1.10", min_gpus=1)), ], ) def test_boring_lite_model_single_device(precision, strategy, devices, accelerator, tmpdir): From af400092d3087b1439f61d9ca499db97556372ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 28 Oct 2021 17:26:59 +0200 Subject: [PATCH 267/331] add test for autocast --- tests/lite/test_lite.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 6135d67d0d026..60da70c09afe0 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -324,6 +324,25 @@ def test_backward_model_input_required(): lite.backward(loss) +@mock.patch("torch.cuda.is_available") +@mock.patch("torch.cuda.device_count", return_value=2) +@pytest.mark.parametrize( + "plugin_context, precision", + [ + ("pytorch_lightning.plugins.precision.double.DoublePrecisionPlugin.forward_context", 64), + ("pytorch_lightning.plugins.precision.precision_plugin.PrecisionPlugin.forward_context", 32), + ("pytorch_lightning.plugins.precision.native_amp.NativeMixedPrecisionPlugin.forward_context", 16), + ], +) +def test_autocast(_, __, plugin_context, precision): + lite = EmptyLite(gpus=1, precision=precision) + with mock.patch(plugin_context) as context: + context().__enter__.assert_not_called() + with lite.autocast(): + context().__enter__.assert_called() + context().__exit__.assert_called() + + @RunIf(min_gpus=2, deepspeed=True, special=True) def test_deepspeed_multiple_models(): class Lite(LightningLite): From eb9b92ec7abf624722bff5d7dde832c528f2fa6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 28 Oct 2021 17:32:10 +0200 Subject: [PATCH 268/331] simplify test --- tests/lite/test_lite.py | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 60da70c09afe0..0fde335a60bdb 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -14,7 +14,7 @@ from copy import deepcopy from unittest import mock -from unittest.mock import Mock, PropertyMock +from unittest.mock import Mock, PropertyMock, MagicMock import pytest import torch @@ -324,23 +324,14 @@ def test_backward_model_input_required(): lite.backward(loss) -@mock.patch("torch.cuda.is_available") -@mock.patch("torch.cuda.device_count", return_value=2) -@pytest.mark.parametrize( - "plugin_context, precision", - [ - ("pytorch_lightning.plugins.precision.double.DoublePrecisionPlugin.forward_context", 64), - ("pytorch_lightning.plugins.precision.precision_plugin.PrecisionPlugin.forward_context", 32), - ("pytorch_lightning.plugins.precision.native_amp.NativeMixedPrecisionPlugin.forward_context", 16), - ], -) -def test_autocast(_, __, plugin_context, precision): - lite = EmptyLite(gpus=1, precision=precision) - with mock.patch(plugin_context) as context: - context().__enter__.assert_not_called() - with lite.autocast(): - context().__enter__.assert_called() - context().__exit__.assert_called() +def test_autocast(): + lite = EmptyLite() + lite._precision_plugin.forward_context = MagicMock() + + lite._precision_plugin.forward_context().__enter__.assert_not_called() + with lite.autocast(): + lite._precision_plugin.forward_context().__enter__.assert_called() + lite._precision_plugin.forward_context().__exit__.assert_called() @RunIf(min_gpus=2, deepspeed=True, special=True) From 3e261e100c5cd6fe8203599effb2acdd87118c01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 28 Oct 2021 17:33:14 +0200 Subject: [PATCH 269/331] add test description --- tests/lite/test_lite.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 0fde335a60bdb..301ed6e9dad01 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -325,6 +325,7 @@ def test_backward_model_input_required(): def test_autocast(): + """Test that the Lite autocast context manager lets the precision plugin handle casting.""" lite = EmptyLite() lite._precision_plugin.forward_context = MagicMock() From 5754ad744616c1bb4980ca3a7baecd53f8ea7977 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 28 Oct 2021 15:33:30 +0000 Subject: [PATCH 270/331] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/lite/test_lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 301ed6e9dad01..1e10b13f612a7 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -14,7 +14,7 @@ from copy import deepcopy from unittest import mock -from unittest.mock import Mock, PropertyMock, MagicMock +from unittest.mock import MagicMock, Mock, PropertyMock import pytest import torch From 85fe0cf125b708921fbc29f9f284f503d5f242da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 28 Oct 2021 17:44:53 +0200 Subject: [PATCH 271/331] remove "mixed" string support --- pytorch_lightning/lite/wrappers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index fe42778dcb39e..c7705f4f9130a 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -82,7 +82,6 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: method.""" precision = self._accelerator.precision_plugin.precision precision_to_type = { - "mixed": torch.float16, 16: torch.float16, 32: torch.float32, 64: torch.float64, From 91a6b3c27c8f03e64a2bc7b657c6dd287c4f3551 Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Thu, 28 Oct 2021 17:48:34 +0200 Subject: [PATCH 272/331] More mixed references --- tests/lite/test_wrappers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/lite/test_wrappers.py b/tests/lite/test_wrappers.py index faed290b75629..9750b8f30da6c 100644 --- a/tests/lite/test_wrappers.py +++ b/tests/lite/test_wrappers.py @@ -41,7 +41,6 @@ def test_lite_module_wraps(): (32, torch.float64, torch.float32), (16, torch.float32, torch.float16), (16, torch.float64, torch.float16), - # ("mixed", torch.float32, torch.float16), # TODO: support precision="mixed" ], ) def test_lite_module_forward_conversion(precision, input_type, expected_type): @@ -50,7 +49,7 @@ def test_lite_module_forward_conversion(precision, input_type, expected_type): device = torch.device("cuda", 0) def check_autocast(forward_input): - assert precision not in (16, "mixed") or torch.is_autocast_enabled() + assert precision != 16 or torch.is_autocast_enabled() return forward_input module = Mock(wraps=torch.nn.Linear(1, 1), side_effect=check_autocast) From f45a97ad36a95bcb8885378dc1b99ffcd5aaefec Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Thu, 28 Oct 2021 18:00:14 +0200 Subject: [PATCH 273/331] Implement `seed_everything` --- pytorch_lightning/lite/lite.py | 17 +++++++++++++++-- tests/lite/test_lite.py | 7 +++---- tests/lite/test_parity.py | 9 ++++----- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index d913ea580f912..9e91e07d0e1f5 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -24,8 +24,7 @@ from torch.optim import Optimizer from torch.utils.data import DataLoader, DistributedSampler, RandomSampler, SequentialSampler -from pytorch_lightning import Trainer -from pytorch_lightning.accelerators import Accelerator +from pytorch_lightning.accelerators.accelerator import Accelerator from pytorch_lightning.lite.wrappers import _LiteDataLoader, _LiteModule, _LiteOptimizer from pytorch_lightning.plugins import ( DDPShardedPlugin, @@ -37,10 +36,12 @@ ) from pytorch_lightning.trainer.connectors.accelerator_connector import AcceleratorConnector from pytorch_lightning.trainer.data_loading import TrainerDataLoadingMixin +from pytorch_lightning.trainer.trainer import Trainer from pytorch_lightning.utilities import DeviceType, DistributedType, move_data_to_device from pytorch_lightning.utilities.apply_func import apply_to_collection, convert_to_tensors from pytorch_lightning.utilities.data import has_iterable_dataset from pytorch_lightning.utilities.exceptions import MisconfigurationException +from pytorch_lightning.utilities.seed import seed_everything class LightningLite(ABC): @@ -371,6 +372,18 @@ def load(self, filepath: Union[str, Path]) -> Any: """ return self._strategy.load_checkpoint(filepath) + @staticmethod + def seed_everything(seed: Optional[int] = None, workers: Optional[bool] = None) -> int: + """Helper function to seed everything without explicitly importing Lightning. + + See :func:`pytorch_lightning.seed_everything` for more details. + """ + if workers is None: + # Lightning sets `workers=False` by default to avoid breaking reproducibility, but since this is a new + # release, we can afford to do it. + workers = True + return seed_everything(seed=seed, workers=workers) + def _run_impl(self, run_method: Callable, *args: Any, **kwargs: Any) -> Any: self._set_plugin_specific_precision_variables() self._accelerator.setup_environment() diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index 1e10b13f612a7..f0c760d3c4a16 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -23,7 +23,6 @@ from torch import nn from torch.utils.data import DataLoader, DistributedSampler, Sampler -from pytorch_lightning import seed_everything from pytorch_lightning.lite import LightningLite from pytorch_lightning.lite.wrappers import _LiteDataLoader, _LiteModule, _LiteOptimizer from pytorch_lightning.plugins import DeepSpeedPlugin, PrecisionPlugin, TrainingTypePlugin @@ -354,11 +353,11 @@ def run(self): for mw_b, mw_a in zip(state_dict.values(), model.state_dict().values()): assert not torch.equal(mw_b, mw_a) - seed_everything(42) + self.seed_everything(42) model_1 = BoringModel() optimizer_1 = torch.optim.SGD(model_1.parameters(), lr=0.0001) - seed_everything(42) + self.seed_everything(42) model_2 = BoringModel() optimizer_2 = torch.optim.SGD(model_2.parameters(), lr=0.0001) @@ -368,7 +367,7 @@ def run(self): model_1, optimizer_1 = self.setup(model_1, optimizer_1) model_2, optimizer_2 = self.setup(model_2, optimizer_2) - seed_everything(42) + self.seed_everything(42) data_list = [] for _ in range(2): optimizer_1.zero_grad() diff --git a/tests/lite/test_parity.py b/tests/lite/test_parity.py index 2c7c58d249a13..b1578f3f47232 100644 --- a/tests/lite/test_parity.py +++ b/tests/lite/test_parity.py @@ -27,7 +27,6 @@ from torch.utils.data import DataLoader from torch.utils.data.distributed import DistributedSampler -from pytorch_lightning import seed_everything from pytorch_lightning.lite import LightningLite from pytorch_lightning.plugins.environments.lightning_environment import find_free_network_port from pytorch_lightning.plugins.training_type.ddp_spawn import DDPSpawnPlugin @@ -116,7 +115,7 @@ def precision_context(precision, accelerator) -> Generator[None, None, None]: ], ) def test_boring_lite_model_single_device(precision, strategy, devices, accelerator, tmpdir): - seed_everything(42) + LightningLite.seed_everything(42) train_dataloader = DataLoader(RandomDataset(32, 8)) model = BoringModel() num_epochs = 1 @@ -168,7 +167,7 @@ def run(rank, model, train_dataloader, num_epochs, precision, accelerator, tmpdi ], ) def test_boring_lite_model_ddp_spawn(precision, strategy, devices, accelerator, tmpdir): - seed_everything(42) + LightningLite.seed_everything(42) train_dataloader = DataLoader(RandomDataset(32, 8)) model = BoringModel() num_epochs = 1 @@ -199,7 +198,7 @@ def test_boring_lite_model_ddp_spawn(precision, strategy, devices, accelerator, ], ) def test_boring_lite_model_ddp(precision, strategy, devices, accelerator, tmpdir): - seed_everything(42) + LightningLite.seed_everything(42) train_dataloader = DataLoader(RandomDataset(32, 4)) model = BoringModel() num_epochs = 1 @@ -213,7 +212,7 @@ def test_boring_lite_model_ddp(precision, strategy, devices, accelerator, tmpdir for w_pure, w_lite in zip(state_dict.values(), lite_model_state_dict.values()): assert not torch.equal(w_pure.cpu(), w_lite.cpu()) - seed_everything(42) + LightningLite.seed_everything(42) train_dataloader = DataLoader(RandomDataset(32, 4)) model = BoringModel() run(lite.global_rank, model, train_dataloader, num_epochs, precision, accelerator, tmpdir) From ba7ac5f311b9083fa0ba22f463e86706b91b68ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 28 Oct 2021 18:30:26 +0200 Subject: [PATCH 274/331] add isinstance check --- pytorch_lightning/accelerators/accelerator.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pytorch_lightning/accelerators/accelerator.py b/pytorch_lightning/accelerators/accelerator.py index a8acec23c6ed3..d2b44fc3fca1c 100644 --- a/pytorch_lightning/accelerators/accelerator.py +++ b/pytorch_lightning/accelerators/accelerator.py @@ -333,6 +333,11 @@ def optimizer_step( """ model = model or self.lightning_module self.precision_plugin.optimizer_step(model, optimizer, opt_idx, lambda_closure, **kwargs) + + if not isinstance(model, pl.LightningModule): + # gradient clipping and norm tracking only available with a LightingModule/Trainer + return + trainer = model.trainer assert isinstance(trainer, pl.Trainer) # TODO: this is done for the entire model but should be changed to per-optimizer From f04b39877fe0e9b6810e77b0a4728ae41c41ab06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 28 Oct 2021 18:35:02 +0200 Subject: [PATCH 275/331] add bfloat16 --- pytorch_lightning/lite/wrappers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index c7705f4f9130a..d2508f6527745 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -82,6 +82,7 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: method.""" precision = self._accelerator.precision_plugin.precision precision_to_type = { + "bf16": torch.bfloat16, 16: torch.float16, 32: torch.float32, 64: torch.float64, From 229b02405a055edfd95720025e77a8a1011f6e58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 28 Oct 2021 18:41:49 +0200 Subject: [PATCH 276/331] rename params_on_cpu --- pytorch_lightning/lite/lite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 9e91e07d0e1f5..17dbf47401b71 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -412,12 +412,12 @@ def _move_model_to_device(self, model: nn.Module, optimizers: List[Optimizer]) - # When the user creates the optimizer, they reference the parameters on the CPU. # However, when running with TPU the parameters get copied and the reference in the optimizer # remains invalid. We need to update the references to point to the parameter tensors on the device. - params_on_cpu = dict(model.named_parameters()) + params_before_move = dict(model.named_parameters()) model = self.to_device(model) # XLA makes a copy on the parameters, so the device is not the same before and after to_device. params_on_device = dict(model.named_parameters()) - mapping = {param: params_on_device[name] for name, param in params_on_cpu.items()} + mapping = {param: params_on_device[name] for name, param in params_before_move.items()} for optimizer in optimizers: for param_group in optimizer.param_groups: param_group["params"] = [mapping.get(p, p) for p in param_group["params"]] From 95db246e8cb89125f820ead9a2944401f7d0c0e4 Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Thu, 28 Oct 2021 20:10:03 +0200 Subject: [PATCH 277/331] Pass down the barrier name --- pytorch_lightning/lite/lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 17dbf47401b71..7b3ae24ee8b8a 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -326,7 +326,7 @@ def barrier(self, name: Optional[str] = None) -> None: # now all processes can read the files and start training """ - self._strategy.barrier() + self._strategy.barrier(name=name) def all_gather( self, data: Union[torch.Tensor, Dict, List, Tuple], group: Optional[Any] = None, sync_grads: bool = False From 0c8e9141f8310750a410362ee6afb97fb63c0d16 Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Thu, 28 Oct 2021 20:11:13 +0200 Subject: [PATCH 278/331] Add back __del__ --- pytorch_lightning/lite/wrappers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index d2508f6527745..c97c280699b05 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -38,7 +38,9 @@ def __init__(self, optimizer: Optimizer, accelerator: Accelerator) -> None: optimizer: The optimizer to wrap accelerator: Reference to the accelerator for handling the optimizer step """ - self.__dict__ = {k: v for k, v in optimizer.__dict__.items() if k not in ("step",)} + # `__del__` is skipped in case the optimizer has implemented custom destructor logic which we would + # not want to call on desturction of the `_LiteOptimizer` + self.__dict__ = {k: v for k, v in optimizer.__dict__.items() if k not in ("step", "__del__")} self.__class__ = type("Lite" + optimizer.__class__.__name__, (self.__class__, optimizer.__class__), {}) self._optimizer = optimizer self._accelerator = accelerator From a93278d68bd347d5e47953e41c59888b64467f01 Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Thu, 28 Oct 2021 20:52:21 +0200 Subject: [PATCH 279/331] Fix mypy --- pytorch_lightning/lite/lite.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 7b3ae24ee8b8a..16898e73d2d79 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -143,12 +143,10 @@ def is_global_zero(self) -> bool: return self._strategy.is_global_zero @abstractmethod - def run(self, *args: Any, **kwargs: Any) -> Any: + def run(self) -> Any: """All the code inside this run method gets accelerated by Lite. - Args: - *args: Add any positional arguments you need, e.g., the hyperparameters for your model. - **kwargs: Add any keyword arguments you need, e.g., the hyperparameters for your model. + You can pass arbitrary arguments to this function when overriding it. """ def setup( @@ -156,7 +154,7 @@ def setup( model: nn.Module, *optimizers: Optimizer, move_to_device: bool = True, - ) -> Union[_LiteModule, List[Union[_LiteModule, _LiteOptimizer]]]: + ) -> Any: # no specific return because the way we want our API to look does not play well with mypy """Setup a model and its optimizers for accelerated training. Args: @@ -179,6 +177,7 @@ def setup( optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self._accelerator) for optimizer in optimizers] self._models_setup += 1 if optimizers: + # join both types in a list for API convenience return [model] + optimizers # type: ignore return model From 65e289b625acc4292424f83cf1e0085b99ef4acc Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Thu, 28 Oct 2021 21:00:16 +0200 Subject: [PATCH 280/331] Fix test --- pytorch_lightning/lite/wrappers.py | 2 +- tests/lite/test_wrappers.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index c97c280699b05..370e84fa10940 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -39,7 +39,7 @@ def __init__(self, optimizer: Optimizer, accelerator: Accelerator) -> None: accelerator: Reference to the accelerator for handling the optimizer step """ # `__del__` is skipped in case the optimizer has implemented custom destructor logic which we would - # not want to call on desturction of the `_LiteOptimizer` + # not want to call on destruction of the `_LiteOptimizer self.__dict__ = {k: v for k, v in optimizer.__dict__.items() if k not in ("step", "__del__")} self.__class__ = type("Lite" + optimizer.__class__.__name__, (self.__class__, optimizer.__class__), {}) self._optimizer = optimizer diff --git a/tests/lite/test_wrappers.py b/tests/lite/test_wrappers.py index 9750b8f30da6c..cbb359a4043ae 100644 --- a/tests/lite/test_wrappers.py +++ b/tests/lite/test_wrappers.py @@ -101,5 +101,3 @@ def test_lite_optimizer_steps(): lite_optimizer.step() accelerator.optimizer_step.assert_called_once() accelerator.optimizer_step.assert_called_with(optimizer, opt_idx=0, lambda_closure=ANY, model=accelerator.model) - lite_optimizer.zero_grad() - optimizer.zero_grad.assert_called_once() From d40822870d8e07e834e89bcba8cfe9c3ff3c82f1 Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Thu, 28 Oct 2021 21:07:22 +0200 Subject: [PATCH 281/331] Add worker init fn --- pytorch_lightning/lite/lite.py | 4 ++++ pytorch_lightning/trainer/data_loading.py | 9 +++++---- tests/trainer/test_dataloaders.py | 8 ++++---- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index 16898e73d2d79..d349672c4d176 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -239,6 +239,10 @@ def _setup_dataloader( dataloader = DataLoader(**kwargs) else: dataloader = _LiteDataLoader(device=device, **kwargs) + + # add worker_init_fn for correct seeding in worker processes + TrainerDataLoadingMixin._auto_add_worker_init_fn(dataloader, self.global_rank) + return self._strategy.process_dataloader(dataloader) def backward(self, tensor: Tensor, *args: Any, model: Optional[_LiteModule] = None, **kwargs: Any) -> None: diff --git a/pytorch_lightning/trainer/data_loading.py b/pytorch_lightning/trainer/data_loading.py index 24206b8af1fc1..726336820b28a 100644 --- a/pytorch_lightning/trainer/data_loading.py +++ b/pytorch_lightning/trainer/data_loading.py @@ -114,9 +114,10 @@ def _worker_check(self, dataloader: DataLoader, name: str) -> None: " in the `DataLoader` init to improve performance." ) - def auto_add_worker_init_fn(self, dataloader: DataLoader) -> None: + @staticmethod + def _auto_add_worker_init_fn(dataloader: DataLoader, rank: int) -> None: if int(os.environ.get("PL_SEED_WORKERS", 0)) and dataloader.worker_init_fn is None: - dataloader.worker_init_fn = partial(pl_worker_init_function, rank=self.global_rank) + dataloader.worker_init_fn = partial(pl_worker_init_function, rank=rank) def _requires_distributed_sampler(self, dataloader) -> bool: return ( @@ -336,7 +337,7 @@ def reset_train_dataloader(self, model: Optional["pl.LightningModule"] = None) - apply_to_collection(self.train_dataloader, DataLoader, self._worker_check, "train_dataloader") # add worker_init_fn for correct seeding in worker processes - apply_to_collection(self.train_dataloader, DataLoader, self.auto_add_worker_init_fn) + apply_to_collection(self.train_dataloader, DataLoader, self._auto_add_worker_init_fn, rank=self.global_rank) # add collate_fn to collect metadata for fault tolerant training if _fault_tolerant_training(): @@ -443,7 +444,7 @@ def _reset_eval_dataloader( dataloaders = [self.prepare_dataloader(dl, False, mode=mode) for dl in dataloaders if dl is not None] # add worker_init_fn for correct seeding in worker processes - apply_to_collection(dataloaders, dtype=DataLoader, function=self.auto_add_worker_init_fn) + apply_to_collection(dataloaders, dtype=DataLoader, function=self._auto_add_worker_init_fn) loader_num_batches = [] diff --git a/tests/trainer/test_dataloaders.py b/tests/trainer/test_dataloaders.py index 2793c71560a81..2e8d552083b99 100644 --- a/tests/trainer/test_dataloaders.py +++ b/tests/trainer/test_dataloaders.py @@ -771,24 +771,24 @@ def test_auto_add_worker_init_fn(): trainer = Trainer() # without pl.seed_everything() - trainer.auto_add_worker_init_fn(dataloader) + trainer._auto_add_worker_init_fn(dataloader) assert dataloader.worker_init_fn is None # with forcefully avoiding it seed_everything(0, workers=False) - trainer.auto_add_worker_init_fn(dataloader) + trainer._auto_add_worker_init_fn(dataloader) assert dataloader.worker_init_fn is None # when user already has a worker_init_fn user_function = _user_worker_init_fn dataloader.worker_init_fn = user_function - trainer.auto_add_worker_init_fn(dataloader) + trainer._auto_add_worker_init_fn(dataloader) assert dataloader.worker_init_fn is user_function dataloader.worker_init_fn = None # main use case seed_everything(0, workers=True) - trainer.auto_add_worker_init_fn(dataloader) + trainer._auto_add_worker_init_fn(dataloader) assert dataloader.worker_init_fn is not None From 952e11c1b7c3acd8f0f8ccd91e121db4fc898b9f Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Thu, 28 Oct 2021 21:15:59 +0200 Subject: [PATCH 282/331] Forgot to pass the global rank --- tests/trainer/test_dataloaders.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/trainer/test_dataloaders.py b/tests/trainer/test_dataloaders.py index 2e8d552083b99..ea31dbaf7d0a1 100644 --- a/tests/trainer/test_dataloaders.py +++ b/tests/trainer/test_dataloaders.py @@ -771,24 +771,24 @@ def test_auto_add_worker_init_fn(): trainer = Trainer() # without pl.seed_everything() - trainer._auto_add_worker_init_fn(dataloader) + trainer._auto_add_worker_init_fn(dataloader, 0) assert dataloader.worker_init_fn is None # with forcefully avoiding it seed_everything(0, workers=False) - trainer._auto_add_worker_init_fn(dataloader) + trainer._auto_add_worker_init_fn(dataloader, 0) assert dataloader.worker_init_fn is None # when user already has a worker_init_fn user_function = _user_worker_init_fn dataloader.worker_init_fn = user_function - trainer._auto_add_worker_init_fn(dataloader) + trainer._auto_add_worker_init_fn(dataloader, 0) assert dataloader.worker_init_fn is user_function dataloader.worker_init_fn = None # main use case seed_everything(0, workers=True) - trainer._auto_add_worker_init_fn(dataloader) + trainer._auto_add_worker_init_fn(dataloader, 0) assert dataloader.worker_init_fn is not None From 50d51248c3955734f26688cfba05e131728eac46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 28 Oct 2021 21:16:02 +0200 Subject: [PATCH 283/331] add back skip of expensive spawn test --- tests/lite/test_parity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lite/test_parity.py b/tests/lite/test_parity.py index b1578f3f47232..bec9339ec8e2f 100644 --- a/tests/lite/test_parity.py +++ b/tests/lite/test_parity.py @@ -158,7 +158,7 @@ def run(rank, model, train_dataloader, num_epochs, precision, accelerator, tmpdi atomic_save(model.state_dict(), os.path.join(tmpdir, "model_spawn.pt")) -# @pytest.mark.skipif(True, reason="Skipping as it takes 80 seconds.") +@pytest.mark.skipif(True, reason="Skipping as it takes 80 seconds.") @RunIf(min_gpus=2) @pytest.mark.parametrize( "precision, strategy, devices, accelerator", From 13fb58a763f1fe372cc5655f129982fb8d2be999 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 28 Oct 2021 21:19:05 +0200 Subject: [PATCH 284/331] resolve todo in _LiteModule --- pytorch_lightning/lite/lite.py | 2 +- pytorch_lightning/lite/wrappers.py | 12 ++++++------ tests/lite/test_wrappers.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pytorch_lightning/lite/lite.py b/pytorch_lightning/lite/lite.py index d349672c4d176..7d0ff6a436b61 100644 --- a/pytorch_lightning/lite/lite.py +++ b/pytorch_lightning/lite/lite.py @@ -173,7 +173,7 @@ def setup( # Let accelerator/plugin wrap and connect the models and optimizers model, optimizers = self._strategy._setup_model_and_optimizers(model, list(optimizers)) - model = _LiteModule(model, self._accelerator) + model = _LiteModule(model, self._precision_plugin) optimizers = [_LiteOptimizer(optimizer=optimizer, accelerator=self._accelerator) for optimizer in optimizers] self._models_setup += 1 if optimizers: diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 370e84fa10940..991b86f25085b 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -20,6 +20,7 @@ from torch.utils.data import DataLoader from pytorch_lightning.accelerators import Accelerator +from pytorch_lightning.plugins import PrecisionPlugin from pytorch_lightning.utilities.apply_func import apply_to_collection, move_data_to_device @@ -60,8 +61,7 @@ def step(self, closure: Optional[Callable] = None) -> None: class _LiteModule(nn.Module): - # TODO: Pass in the precision plugin instead of accelerator - def __init__(self, module: nn.Module, accelerator: Accelerator) -> None: + def __init__(self, module: nn.Module, precision_plugin: PrecisionPlugin) -> None: """The LiteModule is a thin wrapper around the :class:`torch.nn.Module` and handles precision / autocast automatically for the forward pass. @@ -69,11 +69,11 @@ def __init__(self, module: nn.Module, accelerator: Accelerator) -> None: Args: module: The module to wrap - accelerator: Reference to the accelerator for handling precision context + precision_plugin: Reference to the precision plugin for handling precision context """ super().__init__() self._module = module - self._accelerator = accelerator + self._precision_plugin = precision_plugin @property def module(self) -> nn.Module: @@ -82,7 +82,7 @@ def module(self) -> nn.Module: def forward(self, *args: Any, **kwargs: Any) -> Any: """Casts all inputs to the right precision and handles autocast for operations in the module forward method.""" - precision = self._accelerator.precision_plugin.precision + precision = self._precision_plugin.precision precision_to_type = { "bf16": torch.bfloat16, 16: torch.float16, @@ -93,7 +93,7 @@ def forward(self, *args: Any, **kwargs: Any) -> Any: to_type = precision_to_type[precision] args, kwargs = apply_to_collection([args, kwargs], function=lambda t: t.to(to_type), dtype=Tensor) - with self._accelerator.precision_plugin.forward_context(): + with self._precision_plugin.forward_context(): output = self.module(*args, **kwargs) output = apply_to_collection(output, function=lambda t: t.to(torch.get_default_dtype()), dtype=Tensor) diff --git a/tests/lite/test_wrappers.py b/tests/lite/test_wrappers.py index cbb359a4043ae..14a443c042601 100644 --- a/tests/lite/test_wrappers.py +++ b/tests/lite/test_wrappers.py @@ -53,7 +53,7 @@ def check_autocast(forward_input): return forward_input module = Mock(wraps=torch.nn.Linear(1, 1), side_effect=check_autocast) - lite_module = _LiteModule(module, lite._accelerator).to(device) + lite_module = _LiteModule(module, lite._precision_plugin).to(device) out = lite_module(torch.rand(1, dtype=input_type, device=device)) assert module.call_args[0][0].dtype == expected_type assert out.dtype == torch.get_default_dtype() From 9a1e93fa1d87c47ff8fa2d8301ae5e43ccd046d1 Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Thu, 28 Oct 2021 21:27:48 +0200 Subject: [PATCH 285/331] Add seed everything test --- tests/lite/test_lite.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/lite/test_lite.py b/tests/lite/test_lite.py index f0c760d3c4a16..916e0aa542b32 100644 --- a/tests/lite/test_lite.py +++ b/tests/lite/test_lite.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import os from copy import deepcopy from unittest import mock from unittest.mock import MagicMock, Mock, PropertyMock @@ -28,6 +28,7 @@ from pytorch_lightning.plugins import DeepSpeedPlugin, PrecisionPlugin, TrainingTypePlugin from pytorch_lightning.utilities import DistributedType from pytorch_lightning.utilities.exceptions import MisconfigurationException +from pytorch_lightning.utilities.seed import pl_worker_init_function from tests.helpers.runif import RunIf @@ -202,6 +203,18 @@ def test_setup_dataloaders_distributed_sampler_not_needed(): assert lite_dataloader.sampler is custom_sampler +@mock.patch.dict(os.environ, {}, clear=True) +def test_seed_everything(): + """Test that seed everything is static and sets the worker init function on the dataloader.""" + EmptyLite.seed_everything(3) + + lite = EmptyLite() + lite_dataloader = lite.setup_dataloaders(DataLoader(Mock())) + + assert lite_dataloader.worker_init_fn.func is pl_worker_init_function + assert os.environ == {"PL_GLOBAL_SEED": "3", "PL_SEED_WORKERS": "1"} + + @pytest.mark.parametrize( "strategy", [ From f47c2ad69bd488f896ca1a243641c67dcb2bce2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 28 Oct 2021 22:29:00 +0200 Subject: [PATCH 286/331] fix type error --- pytorch_lightning/trainer/data_loading.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/trainer/data_loading.py b/pytorch_lightning/trainer/data_loading.py index 726336820b28a..071eead5613b4 100644 --- a/pytorch_lightning/trainer/data_loading.py +++ b/pytorch_lightning/trainer/data_loading.py @@ -444,7 +444,9 @@ def _reset_eval_dataloader( dataloaders = [self.prepare_dataloader(dl, False, mode=mode) for dl in dataloaders if dl is not None] # add worker_init_fn for correct seeding in worker processes - apply_to_collection(dataloaders, dtype=DataLoader, function=self._auto_add_worker_init_fn) + apply_to_collection( + dataloaders, dtype=DataLoader, function=self._auto_add_worker_init_fn, rank=self.global_rank + ) loader_num_batches = [] From d7b430fa20e1c113f307b31d5b48a2d6f54aca83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 29 Oct 2021 10:41:46 +0200 Subject: [PATCH 287/331] Update pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py Co-authored-by: Pietro Lesci <61748653+pietrolesci@users.noreply.github.com> --- .../basic_examples/mnist_examples/image_classifier_2_lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py index bdf55958b6861..ff9de446cf5ef 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py @@ -88,7 +88,7 @@ def run(self, hparams): test_loss = 0 with torch.no_grad(): for data, target in test_loader: - # NOTE: no need to call .to(device) on the data, target" + # NOTE: no need to call .to(device) on the data, target output = model(data) test_loss += F.nll_loss(output, target, reduction="sum").item() From d51c71c34ee7aa2175efe8462bf217d54c037741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 29 Oct 2021 10:41:54 +0200 Subject: [PATCH 288/331] Update pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py Co-authored-by: Pietro Lesci <61748653+pietrolesci@users.noreply.github.com> --- .../basic_examples/mnist_examples/image_classifier_2_lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py index ff9de446cf5ef..f85335d042965 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py @@ -61,7 +61,7 @@ def run(self, hparams): # TRAINING LOOP model.train() for batch_idx, (data, target) in enumerate(train_loader): - # NOTE: no need to call .to(device) on the data, target" + # NOTE: no need to call .to(device) on the data, target optimizer.zero_grad() output = model(data) loss = F.nll_loss(output, target) From ae2fe70476845afaf9a12b0cbbcc0ad9f057b438 Mon Sep 17 00:00:00 2001 From: tchaton Date: Sat, 30 Oct 2021 11:39:48 +0100 Subject: [PATCH 289/331] update examples --- .../mnist_examples/image_classifier_2_lite.py | 6 +++--- .../image_classifier_3_lite_to_lightning.py | 4 ++-- pl_examples/loop_examples/mnist_lite.py | 21 +++++++------------ 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py index f85335d042965..a98f237e7a933 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py @@ -111,10 +111,10 @@ def run(self, hparams): if hparams.dry_run: break - # When using distributed training, use `self.can_save_checkpoint` + # When using distributed training, use `self.save` # to ensure the current process is allowed to save a checkpoint - if hparams.save_model and self.can_save_checkpoint: - torch.save(model.state_dict(), "mnist_cnn.pt") + if hparams.save_model: + self.save(model.state_dict(), "mnist_cnn.pt") if __name__ == "__main__": diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py index ae38f6a1c8e78..0903187266293 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py @@ -87,8 +87,8 @@ def run(self, hparams): if hparams.dry_run: break - if hparams.save_model and self.can_save_checkpoint: - torch.save(model.state_dict(), "mnist_cnn.pt") + if hparams.save_model: + self.save(model.state_dict(), "mnist_cnn.pt") # Functions for the `LightningModule` conversion diff --git a/pl_examples/loop_examples/mnist_lite.py b/pl_examples/loop_examples/mnist_lite.py index 738964a56f6dc..66a7e17de2a7d 100644 --- a/pl_examples/loop_examples/mnist_lite.py +++ b/pl_examples/loop_examples/mnist_lite.py @@ -144,15 +144,15 @@ def run(self, hparams): train_loader, test_loader = self.setup_dataloaders(train_loader, test_loader) model = Net() - optimizer = optim.Adadelta(model.parameters(), lr=args.lr) + optimizer = optim.Adadelta(model.parameters(), lr=hparams.lr) model, optimizer = self.setup(model, optimizer) - scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) + scheduler = StepLR(optimizer, step_size=1, gamma=hparams.gamma) - MainLoop(self, args, model, optimizer, scheduler, train_loader, test_loader).run() + MainLoop(self, hparams, model, optimizer, scheduler, train_loader, test_loader).run() - if args.save_model and self.is_global_zero: - torch.save(model.state_dict(), "mnist_cnn.pt") + if hparams.save_model: + self.save(model.state_dict(), "mnist_cnn.pt") if __name__ == "__main__": @@ -177,13 +177,8 @@ def run(self, hparams): help="how many batches to wait before logging training status", ) parser.add_argument("--save-model", action="store_true", default=False, help="For Saving the current Model") - args = parser.parse_args() + hparams = parser.parse_args() - seed_everything(args.seed) + seed_everything(hparams.seed) - if torch.cuda.is_available(): - lite_kwargs = {"accelerator": "gpu", "devices": torch.cuda.device_count()} - else: - lite_kwargs = {"accelerator": "cpu"} - - Lite(**lite_kwargs).run(args) + Lite(accelerator="gpu" if torch.cuda.is_available() else "cpu", devices=torch.cuda.device_count()).run(hparams) From 3d4a5efe0562fab747904589cd436bfdb1cd42ad Mon Sep 17 00:00:00 2001 From: tchaton Date: Sat, 30 Oct 2021 11:52:48 +0100 Subject: [PATCH 290/331] update --- pl_examples/README.md | 2 +- .../mnist_examples/image_classifier_2_lite.py | 19 ++++++++++++++++++- .../image_classifier_3_lite_to_lightning.py | 13 +++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/pl_examples/README.md b/pl_examples/README.md index 7093699a25ce4..d405df10a7178 100644 --- a/pl_examples/README.md +++ b/pl_examples/README.md @@ -21,7 +21,7 @@ ______________________________________________________________________ ## Basic Examples -In this folder, we add 2 simple examples: +In this folder, we have 2 simple examples: - [Image Classifier](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/basic_examples/backbone_image_classifier.py) (trains arbitrary datasets with arbitrary backbones). - [Image Classifier + DALI](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/basic_examples/mnist_examples/image_classifier_4_dali.py) (defines the model inside the `LightningModule`). diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py index a98f237e7a933..e63fa731fba04 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py @@ -11,6 +11,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +"""Here are 5 required steps to convert to `LightningLite`. + +1. Subclass `LightningLite` and override its `run` method. + +2. Move the body of your existing `run` function into `run` method. + +3. Remove all ``.to``, ``.cuda`` etc calls since `LightningLite` will take care of it. + +4. Apply `setup` over each model and optimizers pair and `setup_dataloaders` on all your dataloaders +and replace ``loss.backward()`` by ``self.backward(loss)``. + +5. Instantiate your `LightningLite` and call its `run` method. + +Learn more from the documentation: https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.html. +""" + import argparse import torch @@ -23,7 +40,7 @@ from pl_examples.basic_examples.mnist_datamodule import MNIST from pl_examples.basic_examples.mnist_examples.image_classifier_1_pytorch import Net from pytorch_lightning import seed_everything -from pytorch_lightning.lite import LightningLite +from pytorch_lightning.lite import LightningLite # import LightningLite class Lite(LightningLite): diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py index 0903187266293..0bebb498ac2d8 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py @@ -11,6 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +"""Here are the steps to convert from `LightningLite` to a `LightningModule` + +1. Start implementing the `training_step`, `forward`, `train_dataloader` and `configure_optimizers` +functions on the `LightningLite` class. + +2. Utilize those functions within its `run` function. + +3. Finally, switch to `LightningModule` and validate your results are still reproducible (next script). + +Learn more from the documentation: https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.html. +""" + import argparse import torch From 0ffc7d2380a1ee47eddbdfc28a720dfcc3b87460 Mon Sep 17 00:00:00 2001 From: tchaton Date: Sat, 30 Oct 2021 11:53:26 +0100 Subject: [PATCH 291/331] update --- .../basic_examples/mnist_examples/image_classifier_2_lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py index e63fa731fba04..f9417d6eb5f3c 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py @@ -18,7 +18,7 @@ 2. Move the body of your existing `run` function into `run` method. -3. Remove all ``.to``, ``.cuda`` etc calls since `LightningLite` will take care of it. +3. Remove all `.to`, `.cuda` etc calls since `LightningLite` will take care of it. 4. Apply `setup` over each model and optimizers pair and `setup_dataloaders` on all your dataloaders and replace ``loss.backward()`` by ``self.backward(loss)``. From 1b3fb604b11bc5b99da296c7b164610a78dd90ba Mon Sep 17 00:00:00 2001 From: tchaton Date: Sat, 30 Oct 2021 11:55:44 +0100 Subject: [PATCH 292/331] update --- .../basic_examples/mnist_examples/image_classifier_2_lite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py index f9417d6eb5f3c..27e44f6ebaf7e 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py @@ -78,7 +78,7 @@ def run(self, hparams): # TRAINING LOOP model.train() for batch_idx, (data, target) in enumerate(train_loader): - # NOTE: no need to call .to(device) on the data, target + # NOTE: no need to call `.to(device)` on the data, target optimizer.zero_grad() output = model(data) loss = F.nll_loss(output, target) @@ -105,7 +105,7 @@ def run(self, hparams): test_loss = 0 with torch.no_grad(): for data, target in test_loader: - # NOTE: no need to call .to(device) on the data, target + # NOTE: no need to call `.to(device)` on the data, target output = model(data) test_loss += F.nll_loss(output, target, reduction="sum").item() From 357869e26890d6efe709d420a176496f2fe26e40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 1 Nov 2021 11:33:23 +0100 Subject: [PATCH 293/331] replace links with file paths --- pl_examples/README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pl_examples/README.md b/pl_examples/README.md index d405df10a7178..b37755138ee4e 100644 --- a/pl_examples/README.md +++ b/pl_examples/README.md @@ -9,13 +9,13 @@ ______________________________________________________________________ 5 MNIST examples showing how to gradually convert from pure PyTorch to PyTorch Lightning. -The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst) from pure PyTorch is optional but it might helpful to learn about it. +The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst) from pure PyTorch is optional but it might be helpful to learn about it. -- [MNIST with vanilla PyTorch](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/basic_examples/mnist_examples/image_classifier_1_pytorch.py) -- [MNIST with LightningLite](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py) -- [MNIST LightningLite to LightningModule](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py) -- [MNIST with LightningModule](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py) -- [MNIST with LightningModule + LightningDataModule](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py) +- [MNIST with vanilla PyTorch](./basic_examples/mnist_examples/image_classifier_1_pytorch.py) +- [MNIST with LightningLite](./basic_examples/mnist_examples/image_classifier_2_lite.py) +- [MNIST LightningLite to LightningModule](./basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py) +- [MNIST with LightningModule](./basic_examples/mnist_examples/image_classifier_4_lightning.py) +- [MNIST with LightningModule + LightningDataModule](./basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py) ______________________________________________________________________ @@ -23,9 +23,9 @@ ______________________________________________________________________ In this folder, we have 2 simple examples: -- [Image Classifier](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/basic_examples/backbone_image_classifier.py) (trains arbitrary datasets with arbitrary backbones). -- [Image Classifier + DALI](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/basic_examples/mnist_examples/image_classifier_4_dali.py) (defines the model inside the `LightningModule`). -- [Autoencoder](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/basic_examples/autoencoder.py) (shows how the `LightningModule` can be used as a system) +- [Image Classifier](./basic_examples/backbone_image_classifier.py) (trains arbitrary datasets with arbitrary backbones). +- [Image Classifier + DALI](./basic_examples/mnist_examples/image_classifier_4_dali.py) (defines the model inside the `LightningModule`). +- [Autoencoder](./basic_examples/autoencoder.py) (shows how the `LightningModule` can be used as a system) ______________________________________________________________________ From 70067df1a2c979063eb550b82ce2fc45ccfc58b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 1 Nov 2021 11:33:52 +0100 Subject: [PATCH 294/331] fix link --- pl_examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/README.md b/pl_examples/README.md index b37755138ee4e..fd14b6125cc7c 100644 --- a/pl_examples/README.md +++ b/pl_examples/README.md @@ -9,7 +9,7 @@ ______________________________________________________________________ 5 MNIST examples showing how to gradually convert from pure PyTorch to PyTorch Lightning. -The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst) from pure PyTorch is optional but it might be helpful to learn about it. +The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.html) from pure PyTorch is optional but it might be helpful to learn about it. - [MNIST with vanilla PyTorch](./basic_examples/mnist_examples/image_classifier_1_pytorch.py) - [MNIST with LightningLite](./basic_examples/mnist_examples/image_classifier_2_lite.py) From 24f0ff47475cce7baeb29fcb227fea483bdff612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 1 Nov 2021 11:39:59 +0100 Subject: [PATCH 295/331] typos, grammar, fix links --- .../basic_examples/mnist_examples/README.md | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/pl_examples/basic_examples/mnist_examples/README.md b/pl_examples/basic_examples/mnist_examples/README.md index 68028f7059c6a..888d928fa4d43 100644 --- a/pl_examples/basic_examples/mnist_examples/README.md +++ b/pl_examples/basic_examples/mnist_examples/README.md @@ -4,7 +4,7 @@ The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst) from pure PyTorch is optional but it might be helpful to learn about it. -#### 1 . Image Classifier with Vanilla PyTorch +#### 1. Image Classifier with Vanilla PyTorch Trains a simple CNN over MNIST using vanilla PyTorch. @@ -17,22 +17,21 @@ ______________________________________________________________________ #### 2. Image Classifier with LightningLite -This script shows you how to scale the previous script to enable GPU and multi GPU training using [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst). +This script shows you how to scale the previous script to enable GPU and multi-GPU training using [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.html). ```bash -# cpu / multiple gpus if available +# CPU / multiple GPUs if available python image_classifier_2_lite.py ``` ______________________________________________________________________ -#### 3. Image Classifier - Conversion Lite to Lightning +#### 3. Image Classifier - Converting Lite to Lightning -This script shows you to prepare your conversion from [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst) -to `LightningModule`. +This script shows you to prepare your conversion from [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.html) to `LightningModule`. ```bash -# cpu / multiple gpus if available +# CPU / multiple GPUs if available python image_classifier_3_lite_to_lightning.py ``` @@ -40,21 +39,21 @@ ______________________________________________________________________ #### 4. Image Classifier with LightningModule -This script shows you how the result of the conversion to the `LightningModule` and finally get all the benefits from Lightning. +This script shows you the result of the conversion to the `LightningModule` and finally all the benefits you get from Lightning. ```bash -# cpu +# CPU python image_classifier_4_lightning.py -# gpus (any number) +# GPUs (any number) python image_classifier_4_lightning.py --trainer.gpus 2 ``` ______________________________________________________________________ -#### 5. Image Classifier with LightningModule + LightningDataModule +#### 5. Image Classifier with LightningModule and LightningDataModule -This script shows you how extracts the data related components to a `LightningDataModule`. +This script shows you how to extract the data related components into a `LightningDataModule`. ```bash # cpu From 2cd5ba4135aec3b2a930ae639bc969938fbbb621 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 1 Nov 2021 11:40:36 +0100 Subject: [PATCH 296/331] create a sentence --- pl_examples/basic_examples/mnist_examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/basic_examples/mnist_examples/README.md b/pl_examples/basic_examples/mnist_examples/README.md index 888d928fa4d43..2703150fa42b5 100644 --- a/pl_examples/basic_examples/mnist_examples/README.md +++ b/pl_examples/basic_examples/mnist_examples/README.md @@ -1,6 +1,6 @@ ## MNIST Examples -5 MNIST examples showing how to gradually convert from pure PyTorch to PyTorch Lightning. +Here are 5 MNIST examples showing you how to gradually convert from pure PyTorch to PyTorch Lightning. The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst) from pure PyTorch is optional but it might be helpful to learn about it. From 99acc1e02c2f692da61c8515a7a8386a0cfdc92c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 1 Nov 2021 11:45:41 +0100 Subject: [PATCH 297/331] duplicate fixes --- pl_examples/basic_examples/README.md | 59 +++++++------------ .../basic_examples/mnist_examples/README.md | 4 +- 2 files changed, 23 insertions(+), 40 deletions(-) diff --git a/pl_examples/basic_examples/README.md b/pl_examples/basic_examples/README.md index b58632cf51158..e12591c36899a 100644 --- a/pl_examples/basic_examples/README.md +++ b/pl_examples/basic_examples/README.md @@ -4,86 +4,69 @@ Use these examples to test how Lightning works. ## MNIST Examples -5 MNIST examples showing how to gradually convert from pure PyTorch to PyTorch Lightning. +Here are 5 MNIST examples showing you how to gradually convert from pure PyTorch to PyTorch Lightning. The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst) from pure PyTorch is optional but it might be helpful to learn about it. -#### 1 . Image Classifier with Vanilla PyTorch +#### 1. Image Classifier with Vanilla PyTorch Trains a simple CNN over MNIST using vanilla PyTorch. ```bash # cpu -python mnist_examples/image_classifier_1_pytorch.py +python image_classifier_1_pytorch.py ``` ______________________________________________________________________ #### 2. Image Classifier with LightningLite -This script shows you how to scale the previous script to enable GPU and multi GPU training using [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst). +This script shows you how to scale the previous script to enable GPU and multi-GPU training using [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.html). ```bash -# cpu / multiple gpus if available -python mnist_examples/image_classifier_2_lite.py +# CPU / multiple GPUs if available +python image_classifier_2_lite.py ``` ______________________________________________________________________ -#### 3. Image Classifier - Conversion Lite to Lightning +#### 3. Image Classifier - Conversion from Lite to Lightning -This script shows you to prepare your conversion from [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst) -to `LightningModule`. +This script shows you how to prepare your conversion from [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.html) to `LightningModule`. ```bash -# cpu / multiple gpus if available -python mnist_examples/image_classifier_3_lite_to_lightning.py +# CPU / multiple GPUs if available +python image_classifier_3_lite_to_lightning.py ``` ______________________________________________________________________ #### 4. Image Classifier with LightningModule -This script shows you how the result of the conversion to the `LightningModule` and finally get all the benefits from Lightning. +This script shows you the result of the conversion to the `LightningModule` and finally all the benefits you get from Lightning. ```bash -# cpu -python mnist_examples/image_classifier_4_lightning.py +# CPU +python image_classifier_4_lightning.py -# gpus (any number) -python mnist_examples/image_classifier_4_lightning.py --trainer.gpus 2 +# GPUs (any number) +python image_classifier_4_lightning.py --trainer.gpus 2 ``` ______________________________________________________________________ -#### 5. Image Classifier with LightningModule + LightningDataModule +#### 5. Image Classifier with LightningModule and LightningDataModule -This script shows you how extracts the data related components to a `LightningDataModule`. +This script shows you how to extract the data related components into a `LightningDataModule`. ```bash # cpu -python mnist_examples/image_classifier_5_lightning_datamodule.py +python image_classifier_5_lightning_datamodule.py # gpus (any number) -python mnist_examples/image_classifier_5_lightning_datamodule.py --trainer.gpus 2 +python image_classifier_5_lightning_datamodule.py --trainer.gpus 2 -# data parallel -python mnist_examples/image_classifier_5_lightning_datamodule.py --trainer.gpus 2 --trainer.accelerator 'dp' +# dataparallel +python image_classifier_5_lightning_datamodule.py --trainer.gpus 2 --trainer.accelerator 'dp' ``` -______________________________________________________________________ - -#### Autoencoder - -Showing the power of a system... arbitrarily complex training loops - -```bash -# cpu -python autoencoder.py - -# gpus (any number) -python autoencoder.py --trainer.gpus 2 - -# Distributed Data Parallel -python autoencoder.py --trainer.gpus 2 --trainer.accelerator ddp -``` diff --git a/pl_examples/basic_examples/mnist_examples/README.md b/pl_examples/basic_examples/mnist_examples/README.md index 2703150fa42b5..b246189299cbb 100644 --- a/pl_examples/basic_examples/mnist_examples/README.md +++ b/pl_examples/basic_examples/mnist_examples/README.md @@ -26,9 +26,9 @@ python image_classifier_2_lite.py ______________________________________________________________________ -#### 3. Image Classifier - Converting Lite to Lightning +#### 3. Image Classifier - Conversion from Lite to Lightning -This script shows you to prepare your conversion from [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.html) to `LightningModule`. +This script shows you how to prepare your conversion from [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.html) to `LightningModule`. ```bash # CPU / multiple GPUs if available From b5e9a94955e667c08c4560b6dd0d33fa167712c0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Nov 2021 11:11:53 +0000 Subject: [PATCH 298/331] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pl_examples/basic_examples/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/pl_examples/basic_examples/README.md b/pl_examples/basic_examples/README.md index e12591c36899a..399c99116be1c 100644 --- a/pl_examples/basic_examples/README.md +++ b/pl_examples/basic_examples/README.md @@ -69,4 +69,3 @@ python image_classifier_5_lightning_datamodule.py --trainer.gpus 2 # dataparallel python image_classifier_5_lightning_datamodule.py --trainer.gpus 2 --trainer.accelerator 'dp' ``` - From 3e100f78caa334cf32c728a65c6bb19e10ecd87d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 1 Nov 2021 13:18:40 +0100 Subject: [PATCH 299/331] fix changelog --- CHANGELOG.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e99498a5d27f..5c4700115ea77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -119,9 +119,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). * Updated precision attributes in `DeepSpeedPlugin` ([#10164](https://github.com/PyTorchLightning/pytorch-lightning/pull/10164)) * Added the ability to return a result from rank 0 in `DDPSpawnPlugin.spawn` ([#10162](https://github.com/PyTorchLightning/pytorch-lightning/pull/10162)) * Added `pytorch_lightning.lite` package ([#10175](https://github.com/PyTorchLightning/pytorch-lightning/pull/10175)) - * Add `LightningLite` documentation ([#10043](https://github.com/PyTorchLightning/pytorch-lightning/pull/10043)) - * Add `LightningLite` examples ([#9987](https://github.com/PyTorchLightning/pytorch-lightning/pull/9987)) -- Added `XLACheckpointIO` plugin ([#9972](https://github.com/PyTorchLightning/pytorch-lightning/pull/9972)) + * Added `LightningLite` documentation ([#10043](https://github.com/PyTorchLightning/pytorch-lightning/pull/10043)) + * Added `LightningLite` examples ([#9987](https://github.com/PyTorchLightning/pytorch-lightning/pull/9987)) - Added `use_omegaconf` argument to `save_hparams_to_yaml` plugin ([#9170](https://github.com/PyTorchLightning/pytorch-lightning/pull/9170)) - Added `ckpt_path` argument for `Trainer.fit()` ([#10061](https://github.com/PyTorchLightning/pytorch-lightning/pull/10061)) - Added `auto_device_count` method to `Accelerators` ([#10222](https://github.com/PyTorchLightning/pytorch-lightning/pull/10222)) From e01da0bebe20a103b012d5650576dba8a94c31a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 1 Nov 2021 13:33:36 +0100 Subject: [PATCH 300/331] typos and formatting in mnist lite/lightning examples --- .../mnist_examples/image_classifier_2_lite.py | 18 ++++++++--------- .../image_classifier_3_lite_to_lightning.py | 20 +++++++++---------- .../image_classifier_4_lightning.py | 6 +++--- ...image_classifier_5_lightning_datamodule.py | 4 ++-- 4 files changed, 22 insertions(+), 26 deletions(-) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py index 27e44f6ebaf7e..2574d19f9b5c8 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py @@ -12,18 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Here are 5 required steps to convert to `LightningLite`. +"""Here are 5 required steps to convert to LightningLite. -1. Subclass `LightningLite` and override its `run` method. +1. Subclass LightningLite and override its run method. -2. Move the body of your existing `run` function into `run` method. +2. Move the body of your existing ``run`` function into the ``run`` method. -3. Remove all `.to`, `.cuda` etc calls since `LightningLite` will take care of it. +3. Remove all ``.to``, ``.cuda`` etc calls since LightningLite will take care of it. -4. Apply `setup` over each model and optimizers pair and `setup_dataloaders` on all your dataloaders -and replace ``loss.backward()`` by ``self.backward(loss)``. +4. Apply ``setup`` over each model and optimizers pair, ``setup_dataloaders`` on all your dataloaders, +and replace ``loss.backward()`` with ``self.backward(loss)``. -5. Instantiate your `LightningLite` and call its `run` method. +5. Instantiate your LightningLite and call its `run` method. Learn more from the documentation: https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.html. """ @@ -46,7 +46,7 @@ class Lite(LightningLite): def run(self, hparams): self.hparams = hparams - seed_everything(hparams.seed) + seed_everything(hparams.seed) # instead of torch.manual_seed(...) transform = T.Compose([T.ToTensor(), T.Normalize((0.1307,), (0.3081,))]) train_dataset = MNIST("./data", train=True, download=True, transform=transform) @@ -135,7 +135,6 @@ def run(self, hparams): if __name__ == "__main__": - parser = argparse.ArgumentParser(description="LightningLite MNIST Example") parser.add_argument( "--batch-size", type=int, default=64, metavar="N", help="input batch size for training (default: 64)" @@ -143,7 +142,6 @@ def run(self, hparams): parser.add_argument("--epochs", type=int, default=14, metavar="N", help="number of epochs to train (default: 14)") parser.add_argument("--lr", type=float, default=1.0, metavar="LR", help="learning rate (default: 1.0)") parser.add_argument("--gamma", type=float, default=0.7, metavar="M", help="Learning rate step gamma (default: 0.7)") - parser.add_argument("--no-cuda", action="store_true", default=False, help="disables CUDA training") parser.add_argument("--dry-run", action="store_true", default=False, help="quickly check a single pass") parser.add_argument("--seed", type=int, default=1, metavar="S", help="random seed (default: 1)") parser.add_argument( diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py index 0bebb498ac2d8..14746dac248fd 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Here are the steps to convert from `LightningLite` to a `LightningModule` +"""Here are the steps to convert from LightningLite to a LightningModule -1. Start implementing the `training_step`, `forward`, `train_dataloader` and `configure_optimizers` -functions on the `LightningLite` class. +1. Start implementing the ``training_step``, ``forward``, ``train_dataloader`` and ``configure_optimizers`` +methods on the LightningLite class. -2. Utilize those functions within its `run` function. +2. Utilize those methods within the ``run`` method. -3. Finally, switch to `LightningModule` and validate your results are still reproducible (next script). +3. Finally, switch to LightningModule and validate that your results are still reproducible (next script). Learn more from the documentation: https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.html. """ @@ -40,8 +40,7 @@ class Lite(LightningLite): - - """`Lite` is starting to look like a `LightningModule`.""" + """Lite is starting to look like a LightningModule.""" def run(self, hparams): self.hparams = hparams @@ -109,14 +108,14 @@ def forward(self, x): return self.model(x) def training_step(self, batch, batch_idx): - """Here you compute and return the training loss+ compute extra training metrics.""" + """Here you compute and return the training loss and compute extra training metrics.""" x, y = batch logits = self.forward(x) loss = F.nll_loss(logits, y.long()) return loss def test_step(self, batch, batch_idx): - """Here you compute and return the testing loss+ compute extra testing metrics.""" + """Here you compute and return the testing loss and compute extra testing metrics.""" x, y = batch logits = self.forward(x) loss = F.nll_loss(logits, y.long()) @@ -127,7 +126,7 @@ def configure_optimizers(self): optimizer = optim.Adadelta(self.model.parameters(), lr=self.hparams.lr) return [optimizer], [StepLR(optimizer, step_size=1, gamma=self.hparams.gamma)] - # Functions for the `LightningDataModule` conversion + # Methods for the `LightningDataModule` conversion @property def transform(self): @@ -146,7 +145,6 @@ def test_dataloader(self): if __name__ == "__main__": - parser = argparse.ArgumentParser(description="LightningLite to LightningModule MNIST Example") parser.add_argument( "--batch-size", type=int, default=64, metavar="N", help="input batch size for training (default: 64)" diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py b/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py index 42501cdbfdddb..521cd529f125f 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""MNIST simple image classifier example with LightningModule. +"""Simple MNIST image classifier example with LightningModule. To run: python image_classifier_4_lightning.py --trainer.max_epochs=50 """ @@ -55,7 +55,7 @@ def configure_optimizers(self): optimizer = torch.optim.Adadelta(self.model.parameters(), lr=self.hparams.lr) return [optimizer], [torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=self.hparams.gamma)] - # Functions for the `LightningDataModule` conversion + # Methods for the `LightningDataModule` conversion @property def transform(self): @@ -74,7 +74,7 @@ def test_dataloader(self): def cli_main(): - # The LightningCLI removes all the boilerplate associate to arguments parsing. This is purely optional. + # The LightningCLI removes all the boilerplate associated with arguments parsing. This is purely optional. cli = LightningCLI(ImageClassifier, seed_everything_default=42, save_config_overwrite=True, run=False) cli.trainer.fit(cli.model, datamodule=cli.datamodule) cli.trainer.test(ckpt_path="best", datamodule=cli.datamodule) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py b/pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py index 3dfb5543aca21..a6970c30ecbf4 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""MNIST simple image classifier example with LightningModule and DataModule. +"""Simple MNIST image classifier example with LightningModule and LightningDataModule. To run: python image_classifier_5_lightning_datamodule.py --trainer.max_epochs=50 """ @@ -80,7 +80,7 @@ def test_dataloader(self): def cli_main(): - # The LightningCLI removes all the boilerplate associate to arguments parsing. This is purely optional. + # The LightningCLI removes all the boilerplate associated with arguments parsing. This is purely optional. cli = LightningCLI( ImageClassifier, MNISTDataModule, seed_everything_default=42, save_config_overwrite=True, run=False ) From c3859981ca1a236d589f20d02d5a9d4146dd4b1c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Nov 2021 12:34:56 +0000 Subject: [PATCH 301/331] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../mnist_examples/image_classifier_3_lite_to_lightning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py index 14746dac248fd..2aa38fff4969b 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Here are the steps to convert from LightningLite to a LightningModule +"""Here are the steps to convert from LightningLite to a LightningModule. 1. Start implementing the ``training_step``, ``forward``, ``train_dataloader`` and ``configure_optimizers`` methods on the LightningLite class. From eb538848a11a921c39934c48e1caf9058b68ee4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 1 Nov 2021 13:52:45 +0100 Subject: [PATCH 302/331] fixes for loop example mnist_lite --- pl_examples/loop_examples/mnist_lite.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pl_examples/loop_examples/mnist_lite.py b/pl_examples/loop_examples/mnist_lite.py index 66a7e17de2a7d..e636d0a71d122 100644 --- a/pl_examples/loop_examples/mnist_lite.py +++ b/pl_examples/loop_examples/mnist_lite.py @@ -37,6 +37,7 @@ def __init__(self, lite, args, model, optimizer, scheduler, dataloader): self.optimizer = optimizer self.scheduler = scheduler self.dataloader = dataloader + self.dataloader_iter = None @property def done(self) -> bool: @@ -79,7 +80,9 @@ def __init__(self, lite, args, model, dataloader): self.args = args self.model = model self.dataloader = dataloader + self.dataloader_iter = None self.accuracy = Accuracy() + self.test_loss = 0 @property def done(self) -> bool: @@ -166,7 +169,6 @@ def run(self, hparams): parser.add_argument("--epochs", type=int, default=14, metavar="N", help="number of epochs to train (default: 14)") parser.add_argument("--lr", type=float, default=1.0, metavar="LR", help="learning rate (default: 1.0)") parser.add_argument("--gamma", type=float, default=0.7, metavar="M", help="Learning rate step gamma (default: 0.7)") - parser.add_argument("--no-cuda", action="store_true", default=False, help="disables CUDA training") parser.add_argument("--dry-run", action="store_true", default=False, help="quickly check a single pass") parser.add_argument("--seed", type=int, default=1, metavar="S", help="random seed (default: 1)") parser.add_argument( From d32f428f970c798b85f62a4ec64aa4778cac9b48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 1 Nov 2021 16:25:23 +0100 Subject: [PATCH 303/331] undo gitignore changes --- .gitignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitignore b/.gitignore index eaf67251056f1..997886d648614 100644 --- a/.gitignore +++ b/.gitignore @@ -139,8 +139,6 @@ mnist/ legacy/checkpoints/ *.gz *ubyte -grid_generated* -grid_ori* # pl tests From ac75983fc1a6c56af98a35580e0be9e000fd510f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 1 Nov 2021 16:29:00 +0100 Subject: [PATCH 304/331] use strategy arg Co-authored-by: Kaushik B <45285388+kaushikb11@users.noreply.github.com> --- pl_examples/basic_examples/README.md | 4 ++-- tests/special_tests.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pl_examples/basic_examples/README.md b/pl_examples/basic_examples/README.md index 399c99116be1c..3cd2aedfb8e9a 100644 --- a/pl_examples/basic_examples/README.md +++ b/pl_examples/basic_examples/README.md @@ -66,6 +66,6 @@ python image_classifier_5_lightning_datamodule.py # gpus (any number) python image_classifier_5_lightning_datamodule.py --trainer.gpus 2 -# dataparallel -python image_classifier_5_lightning_datamodule.py --trainer.gpus 2 --trainer.accelerator 'dp' +# Distributed Data parallel +python image_classifier_5_lightning_datamodule.py --trainer.gpus 2 --trainer.strategy 'ddp' ``` diff --git a/tests/special_tests.sh b/tests/special_tests.sh index edb6f2980c698..f4b760dd75291 100755 --- a/tests/special_tests.sh +++ b/tests/special_tests.sh @@ -87,7 +87,7 @@ fi # report+="Ran\ttests/plugins/environments/torch_elastic_deadlock.py\n" # test that a user can manually launch individual processes -args="--trainer.gpus 2 --trainer.accelerator ddp --trainer.max_epochs=1 --trainer.limit_train_batches=1 --trainer.limit_val_batches=1 --trainer.limit_test_batches=1" +args="--trainer.gpus 2 --trainer.strategy ddp --trainer.max_epochs=1 --trainer.limit_train_batches=1 --trainer.limit_val_batches=1 --trainer.limit_test_batches=1" MASTER_ADDR="localhost" MASTER_PORT=1234 LOCAL_RANK=1 python pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py ${args} & MASTER_ADDR="localhost" MASTER_PORT=1234 LOCAL_RANK=0 python pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py ${args} report+="Ran\tmanual ddp launch test\n" From b4417454d7fff5df80f778648ae563fcb03a0756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 1 Nov 2021 16:30:25 +0100 Subject: [PATCH 305/331] backticks for run --- .../basic_examples/mnist_examples/image_classifier_2_lite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py index 2574d19f9b5c8..72eef2852029b 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py @@ -14,7 +14,7 @@ """Here are 5 required steps to convert to LightningLite. -1. Subclass LightningLite and override its run method. +1. Subclass LightningLite and override its ``run`` method. 2. Move the body of your existing ``run`` function into the ``run`` method. @@ -23,7 +23,7 @@ 4. Apply ``setup`` over each model and optimizers pair, ``setup_dataloaders`` on all your dataloaders, and replace ``loss.backward()`` with ``self.backward(loss)``. -5. Instantiate your LightningLite and call its `run` method. +5. Instantiate your LightningLite and call its ``run`` method. Learn more from the documentation: https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.html. """ From 8045e0cd779f277861e988bbb1b9e49f48cbc4db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 1 Nov 2021 16:32:33 +0100 Subject: [PATCH 306/331] auto accelerator and devices --- .../basic_examples/mnist_examples/image_classifier_2_lite.py | 2 +- .../mnist_examples/image_classifier_3_lite_to_lightning.py | 2 +- pl_examples/loop_examples/mnist_lite.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py index 72eef2852029b..213e1148f81c4 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py @@ -154,4 +154,4 @@ def run(self, hparams): parser.add_argument("--save-model", action="store_true", default=False, help="For Saving the current Model") hparams = parser.parse_args() - Lite(accelerator="gpu" if torch.cuda.is_available() else "cpu", devices=torch.cuda.device_count()).run(hparams) + Lite(accelerator="auto", devices="auto").run(hparams) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py index 2aa38fff4969b..ee16ad26ff43a 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py @@ -164,4 +164,4 @@ def test_dataloader(self): parser.add_argument("--save-model", action="store_true", default=False, help="For Saving the current Model") hparams = parser.parse_args() - Lite(accelerator="gpu" if torch.cuda.is_available() else "cpu", devices=torch.cuda.device_count()).run(hparams) + Lite(accelerator="auto", devices="auto").run(hparams) diff --git a/pl_examples/loop_examples/mnist_lite.py b/pl_examples/loop_examples/mnist_lite.py index e636d0a71d122..13f68081bc1ba 100644 --- a/pl_examples/loop_examples/mnist_lite.py +++ b/pl_examples/loop_examples/mnist_lite.py @@ -183,4 +183,4 @@ def run(self, hparams): seed_everything(hparams.seed) - Lite(accelerator="gpu" if torch.cuda.is_available() else "cpu", devices=torch.cuda.device_count()).run(hparams) + Lite(accelerator="auto", devices="auto").run(hparams) From 5b6243fc02dd6a025cca0c262e46b0595647e194 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 1 Nov 2021 16:36:10 +0100 Subject: [PATCH 307/331] update comments about dp/strategy --- pl_examples/basic_examples/README.md | 4 ++-- pl_examples/basic_examples/mnist_examples/README.md | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pl_examples/basic_examples/README.md b/pl_examples/basic_examples/README.md index 3cd2aedfb8e9a..46e737c3f3cef 100644 --- a/pl_examples/basic_examples/README.md +++ b/pl_examples/basic_examples/README.md @@ -60,10 +60,10 @@ ______________________________________________________________________ This script shows you how to extract the data related components into a `LightningDataModule`. ```bash -# cpu +# CPU python image_classifier_5_lightning_datamodule.py -# gpus (any number) +# GPUs (any number) python image_classifier_5_lightning_datamodule.py --trainer.gpus 2 # Distributed Data parallel diff --git a/pl_examples/basic_examples/mnist_examples/README.md b/pl_examples/basic_examples/mnist_examples/README.md index b246189299cbb..3e4e5c91bbefa 100644 --- a/pl_examples/basic_examples/mnist_examples/README.md +++ b/pl_examples/basic_examples/mnist_examples/README.md @@ -56,12 +56,12 @@ ______________________________________________________________________ This script shows you how to extract the data related components into a `LightningDataModule`. ```bash -# cpu +# CPU python image_classifier_5_lightning_datamodule.py -# gpus (any number) +# GPUs (any number) python image_classifier_5_lightning_datamodule.py --trainer.gpus 2 -# dataparallel -python image_classifier_5_lightning_datamodule.py --trainer.gpus 2 --trainer.accelerator 'dp' +# Distributed Data parallel +python image_classifier_5_lightning_datamodule.py --trainer.gpus 2 --trainer.strategy 'ddp' ``` From 51ca1e697858bc7a54c86ba7cd10a5e4cd05fa70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 1 Nov 2021 16:55:40 +0100 Subject: [PATCH 308/331] address a couple missed commens from ari --- .../mnist_examples/image_classifier_3_lite_to_lightning.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py index ee16ad26ff43a..12f318cea64dc 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py @@ -44,13 +44,14 @@ class Lite(LightningLite): def run(self, hparams): self.hparams = hparams - seed_everything(hparams.seed) + seed_everything(hparams.seed) # instead of torch.manual_seed(...) self.model = Net() [optimizer], [scheduler] = self.configure_optimizers() model, optimizer = self.setup(self.model, optimizer) if self.is_global_zero: + # In multi-device training, this code will only run on the first process / GPU self.prepare_data() train_loader, test_loader = self.setup_dataloaders(self.train_dataloader(), self.train_dataloader()) @@ -102,7 +103,7 @@ def run(self, hparams): if hparams.save_model: self.save(model.state_dict(), "mnist_cnn.pt") - # Functions for the `LightningModule` conversion + # Methods for the `LightningModule` conversion def forward(self, x): return self.model(x) From 9fd0bba736123c4e81948034fe45420fda60ac69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 1 Nov 2021 17:53:26 +0100 Subject: [PATCH 309/331] Update pl_examples/basic_examples/README.md Co-authored-by: Jirka Borovec --- pl_examples/basic_examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/basic_examples/README.md b/pl_examples/basic_examples/README.md index 46e737c3f3cef..b14e6d9fcd591 100644 --- a/pl_examples/basic_examples/README.md +++ b/pl_examples/basic_examples/README.md @@ -6,7 +6,7 @@ Use these examples to test how Lightning works. Here are 5 MNIST examples showing you how to gradually convert from pure PyTorch to PyTorch Lightning. -The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst) from pure PyTorch is optional but it might be helpful to learn about it. +The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/en/stable/starter/lightning_lite.rst) from pure PyTorch is optional but it might be helpful to learn about it. #### 1. Image Classifier with Vanilla PyTorch From 60a65dc85d0c88dcfcf9e3144ef83771c02f2da7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 1 Nov 2021 17:54:26 +0100 Subject: [PATCH 310/331] update links, latest -> stable --- pl_examples/basic_examples/README.md | 6 +++--- pl_examples/basic_examples/mnist_examples/README.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pl_examples/basic_examples/README.md b/pl_examples/basic_examples/README.md index 46e737c3f3cef..a5c743093e9f4 100644 --- a/pl_examples/basic_examples/README.md +++ b/pl_examples/basic_examples/README.md @@ -6,7 +6,7 @@ Use these examples to test how Lightning works. Here are 5 MNIST examples showing you how to gradually convert from pure PyTorch to PyTorch Lightning. -The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst) from pure PyTorch is optional but it might be helpful to learn about it. +The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/en/stable/starter/lightning_lite.rst) from pure PyTorch is optional but it might be helpful to learn about it. #### 1. Image Classifier with Vanilla PyTorch @@ -21,7 +21,7 @@ ______________________________________________________________________ #### 2. Image Classifier with LightningLite -This script shows you how to scale the previous script to enable GPU and multi-GPU training using [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.html). +This script shows you how to scale the previous script to enable GPU and multi-GPU training using [LightningLite](https://pytorch-lightning.readthedocs.io/en/stable/starter/lightning_lite.html). ```bash # CPU / multiple GPUs if available @@ -32,7 +32,7 @@ ______________________________________________________________________ #### 3. Image Classifier - Conversion from Lite to Lightning -This script shows you how to prepare your conversion from [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.html) to `LightningModule`. +This script shows you how to prepare your conversion from [LightningLite](https://pytorch-lightning.readthedocs.io/en/stable/starter/lightning_lite.html) to `LightningModule`. ```bash # CPU / multiple GPUs if available diff --git a/pl_examples/basic_examples/mnist_examples/README.md b/pl_examples/basic_examples/mnist_examples/README.md index 3e4e5c91bbefa..de1529002fb56 100644 --- a/pl_examples/basic_examples/mnist_examples/README.md +++ b/pl_examples/basic_examples/mnist_examples/README.md @@ -2,7 +2,7 @@ Here are 5 MNIST examples showing you how to gradually convert from pure PyTorch to PyTorch Lightning. -The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.rst) from pure PyTorch is optional but it might be helpful to learn about it. +The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/stable/lightning_lite.rst) from pure PyTorch is optional but it might be helpful to learn about it. #### 1. Image Classifier with Vanilla PyTorch @@ -17,7 +17,7 @@ ______________________________________________________________________ #### 2. Image Classifier with LightningLite -This script shows you how to scale the previous script to enable GPU and multi-GPU training using [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.html). +This script shows you how to scale the previous script to enable GPU and multi-GPU training using [LightningLite](https://pytorch-lightning.readthedocs.io/en/stable/starter/lightning_lite.html). ```bash # CPU / multiple GPUs if available @@ -28,7 +28,7 @@ ______________________________________________________________________ #### 3. Image Classifier - Conversion from Lite to Lightning -This script shows you how to prepare your conversion from [LightningLite](https://pytorch-lightning.readthedocs.io/en/latest/starter/lightning_lite.html) to `LightningModule`. +This script shows you how to prepare your conversion from [LightningLite](https://pytorch-lightning.readthedocs.io/en/stable/starter/lightning_lite.html) to `LightningModule`. ```bash # CPU / multiple GPUs if available From f33911d709796d1791df63dbad4dfa40a3b852be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 1 Nov 2021 18:10:50 +0100 Subject: [PATCH 311/331] switch order in run_examples.sh --- pl_examples/run_examples.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pl_examples/run_examples.sh b/pl_examples/run_examples.sh index 989c28eef13b7..48a0589c62b7a 100755 --- a/pl_examples/run_examples.sh +++ b/pl_examples/run_examples.sh @@ -15,6 +15,13 @@ python "${dir_path}/basic_examples/backbone_image_classifier.py" ${args} "$@" python "${dir_path}/basic_examples/autoencoder.py" ${args} "$@" +args="--dry-run" +python "${dir_path}/basic_examples/mnist_examples/image_classifier_1_pytorch.py" ${args} +python "${dir_path}/basic_examples/mnist_examples/image_classifier_2_lite.py" ${args} +python "${dir_path}/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py" ${args} +python "${dir_path}/loop_examples/mnist_lite.py" ${args} + + args=" --trainer.max_epochs=1 --trainer.limit_train_batches=2 @@ -25,9 +32,3 @@ args=" python "${dir_path}/basic_examples/mnist_examples/image_classifier_4_lightning.py" ${args} "$@" python "${dir_path}/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py" ${args} "$@" - -args="--dry-run" -python "${dir_path}/basic_examples/mnist_examples/image_classifier_1_pytorch.py" ${args} -python "${dir_path}/basic_examples/mnist_examples/image_classifier_2_lite.py" ${args} -python "${dir_path}/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py" ${args} -python "${dir_path}/loop_examples/mnist_lite.py" ${args} From 6e76183eda5720aab811bb1ba293faa305de289a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 1 Nov 2021 18:11:14 +0100 Subject: [PATCH 312/331] capitalization c --- pl_examples/basic_examples/README.md | 2 +- pl_examples/basic_examples/mnist_examples/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pl_examples/basic_examples/README.md b/pl_examples/basic_examples/README.md index a5c743093e9f4..19931996a9e27 100644 --- a/pl_examples/basic_examples/README.md +++ b/pl_examples/basic_examples/README.md @@ -13,7 +13,7 @@ The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/ Trains a simple CNN over MNIST using vanilla PyTorch. ```bash -# cpu +# CPU python image_classifier_1_pytorch.py ``` diff --git a/pl_examples/basic_examples/mnist_examples/README.md b/pl_examples/basic_examples/mnist_examples/README.md index de1529002fb56..1ee9fc016d827 100644 --- a/pl_examples/basic_examples/mnist_examples/README.md +++ b/pl_examples/basic_examples/mnist_examples/README.md @@ -9,7 +9,7 @@ The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/ Trains a simple CNN over MNIST using vanilla PyTorch. ```bash -# cpu +# CPU python image_classifier_1_pytorch.py ``` From 6c7e630a8e048fe677abdec3164b038bf8c1d106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 1 Nov 2021 18:16:55 +0100 Subject: [PATCH 313/331] Update pl_examples/basic_examples/README.md Co-authored-by: Jirka Borovec --- pl_examples/basic_examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/basic_examples/README.md b/pl_examples/basic_examples/README.md index 19931996a9e27..3393f543583ad 100644 --- a/pl_examples/basic_examples/README.md +++ b/pl_examples/basic_examples/README.md @@ -66,6 +66,6 @@ python image_classifier_5_lightning_datamodule.py # GPUs (any number) python image_classifier_5_lightning_datamodule.py --trainer.gpus 2 -# Distributed Data parallel +# Distributed Data Parallel (DDP) python image_classifier_5_lightning_datamodule.py --trainer.gpus 2 --trainer.strategy 'ddp' ``` From be1d820a2d5ff956a8750752141c341fc209f876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 1 Nov 2021 18:17:44 +0100 Subject: [PATCH 314/331] Update pl_examples/basic_examples/README.md Co-authored-by: Jirka Borovec --- pl_examples/basic_examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/basic_examples/README.md b/pl_examples/basic_examples/README.md index 3393f543583ad..8e6695c9e9dbf 100644 --- a/pl_examples/basic_examples/README.md +++ b/pl_examples/basic_examples/README.md @@ -43,7 +43,7 @@ ______________________________________________________________________ #### 4. Image Classifier with LightningModule -This script shows you the result of the conversion to the `LightningModule` and finally all the benefits you get from Lightning. +This script shows you the result of the conversion to the `LightningModule` and finally all the benefits you get from the Lightning ecosystem. ```bash # CPU From 09ccc0d7ff8b795863b0ce386cce56d09d314143 Mon Sep 17 00:00:00 2001 From: tchaton Date: Mon, 1 Nov 2021 18:43:03 +0000 Subject: [PATCH 315/331] update on comments --- pl_examples/README.md | 4 ++-- pl_examples/basic_examples/README.md | 6 +++--- pl_examples/basic_examples/mnist_examples/README.md | 6 +++--- ...ng.py => image_classifier_3_lite_to_lightning_module.py} | 0 ..._lightning.py => image_classifier_4_lightning_module.py} | 2 +- pl_examples/run_examples.sh | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) rename pl_examples/basic_examples/mnist_examples/{image_classifier_3_lite_to_lightning.py => image_classifier_3_lite_to_lightning_module.py} (100%) rename pl_examples/basic_examples/mnist_examples/{image_classifier_4_lightning.py => image_classifier_4_lightning_module.py} (97%) diff --git a/pl_examples/README.md b/pl_examples/README.md index fd14b6125cc7c..58cc5c64d8c4b 100644 --- a/pl_examples/README.md +++ b/pl_examples/README.md @@ -13,8 +13,8 @@ The transition through [LightningLite](https://pytorch-lightning.readthedocs.io/ - [MNIST with vanilla PyTorch](./basic_examples/mnist_examples/image_classifier_1_pytorch.py) - [MNIST with LightningLite](./basic_examples/mnist_examples/image_classifier_2_lite.py) -- [MNIST LightningLite to LightningModule](./basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py) -- [MNIST with LightningModule](./basic_examples/mnist_examples/image_classifier_4_lightning.py) +- [MNIST LightningLite to LightningModule](./basic_examples/mnist_examples/image_classifier_3_lite_to_lightning_module.py) +- [MNIST with LightningModule](./basic_examples/mnist_examples/image_classifier_4_lightning_module.py) - [MNIST with LightningModule + LightningDataModule](./basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py) ______________________________________________________________________ diff --git a/pl_examples/basic_examples/README.md b/pl_examples/basic_examples/README.md index 8e6695c9e9dbf..b1b02f90ecb24 100644 --- a/pl_examples/basic_examples/README.md +++ b/pl_examples/basic_examples/README.md @@ -36,7 +36,7 @@ This script shows you how to prepare your conversion from [LightningLite](https: ```bash # CPU / multiple GPUs if available -python image_classifier_3_lite_to_lightning.py +python image_classifier_3_lite_to_lightning_module.py ``` ______________________________________________________________________ @@ -47,10 +47,10 @@ This script shows you the result of the conversion to the `LightningModule` and ```bash # CPU -python image_classifier_4_lightning.py +python image_classifier_4_lightning_module.py # GPUs (any number) -python image_classifier_4_lightning.py --trainer.gpus 2 +python image_classifier_4_lightning_module.py --trainer.gpus 2 ``` ______________________________________________________________________ diff --git a/pl_examples/basic_examples/mnist_examples/README.md b/pl_examples/basic_examples/mnist_examples/README.md index 1ee9fc016d827..c82960af1ff22 100644 --- a/pl_examples/basic_examples/mnist_examples/README.md +++ b/pl_examples/basic_examples/mnist_examples/README.md @@ -32,7 +32,7 @@ This script shows you how to prepare your conversion from [LightningLite](https: ```bash # CPU / multiple GPUs if available -python image_classifier_3_lite_to_lightning.py +python image_classifier_3_lite_to_lightning_module.py ``` ______________________________________________________________________ @@ -43,10 +43,10 @@ This script shows you the result of the conversion to the `LightningModule` and ```bash # CPU -python image_classifier_4_lightning.py +python image_classifier_4_lightning_module.py # GPUs (any number) -python image_classifier_4_lightning.py --trainer.gpus 2 +python image_classifier_4_lightning_module.py --trainer.gpus 2 ``` ______________________________________________________________________ diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning_module.py similarity index 100% rename from pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py rename to pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning_module.py diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py b/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning_module.py similarity index 97% rename from pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py rename to pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning_module.py index 521cd529f125f..6ca06f151520b 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning_module.py @@ -13,7 +13,7 @@ # limitations under the License. """Simple MNIST image classifier example with LightningModule. -To run: python image_classifier_4_lightning.py --trainer.max_epochs=50 +To run: python image_classifier_4_lightning_module.py --trainer.max_epochs=50 """ import torch import torchvision.transforms as T diff --git a/pl_examples/run_examples.sh b/pl_examples/run_examples.sh index 48a0589c62b7a..321d39ce674fa 100755 --- a/pl_examples/run_examples.sh +++ b/pl_examples/run_examples.sh @@ -18,7 +18,7 @@ python "${dir_path}/basic_examples/autoencoder.py" ${args} "$@" args="--dry-run" python "${dir_path}/basic_examples/mnist_examples/image_classifier_1_pytorch.py" ${args} python "${dir_path}/basic_examples/mnist_examples/image_classifier_2_lite.py" ${args} -python "${dir_path}/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning.py" ${args} +python "${dir_path}/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning_module.py" ${args} python "${dir_path}/loop_examples/mnist_lite.py" ${args} @@ -30,5 +30,5 @@ args=" --trainer.limit_predict_batches=2 " -python "${dir_path}/basic_examples/mnist_examples/image_classifier_4_lightning.py" ${args} "$@" +python "${dir_path}/basic_examples/mnist_examples/image_classifier_4_lightning_module.py" ${args} "$@" python "${dir_path}/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py" ${args} "$@" From a27d0e3a55e43034df7a7eaf9550a969e5309cbd Mon Sep 17 00:00:00 2001 From: tchaton Date: Mon, 1 Nov 2021 18:50:07 +0000 Subject: [PATCH 316/331] hotfix --- pytorch_lightning/lite/wrappers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index d9acba70bcba1..9ebf32f109223 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -164,6 +164,9 @@ def __init__(self, dataloader: Iterable, device: Optional[torch.device] = None) self._dataloader = dataloader self._device = device + def __len__(self) -> int: + return len(self._dataloader) + @property def device(self) -> Optional[torch.device]: return self._device From 8a970ac3b0c77ab4168b3ddaba5255149fad442a Mon Sep 17 00:00:00 2001 From: tchaton Date: Mon, 1 Nov 2021 18:50:48 +0000 Subject: [PATCH 317/331] update --- tests/lite/test_wrappers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/lite/test_wrappers.py b/tests/lite/test_wrappers.py index 4dd7b4a890648..8fc4f7e9c6e53 100644 --- a/tests/lite/test_wrappers.py +++ b/tests/lite/test_wrappers.py @@ -93,6 +93,8 @@ def test_lite_dataloader_device_placement(src_device, dest_device): batch0 = next(iterator) assert batch0 == 0 + assert len(lite_dataloader) == 4 + def test_lite_optimizer_wraps(): """Test that the LiteOptimizer fully wraps the optimizer.""" From fd3d286ead3048f70fffa1f54c60ba14442b5ff1 Mon Sep 17 00:00:00 2001 From: tchaton Date: Mon, 1 Nov 2021 19:17:31 +0000 Subject: [PATCH 318/331] update --- pytorch_lightning/lite/wrappers.py | 12 ++++++++---- pytorch_lightning/utilities/data.py | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 9ebf32f109223..782512885d20c 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -14,7 +14,7 @@ import functools import inspect from contextlib import contextmanager -from typing import Any, Callable, Dict, Generator, Iterable, Iterator, Optional, Set, Type, Union +from typing import Any, Callable, Dict, Generator, Iterable, Iterator, Optional, Set, Sized, Type, Union import torch from torch import nn as nn @@ -150,7 +150,9 @@ def _replace_dataloader_init_method() -> Generator: class _LiteDataLoader: - def __init__(self, dataloader: Iterable, device: Optional[torch.device] = None) -> None: + def __init__( + self, dataloader: Union[Iterable[Any], DataLoader[Any]], device: Optional[torch.device] = None + ) -> None: """The LiteDataLoader is an extension of an Iterator. It would move the data to the device automatically if the device is specified. @@ -164,8 +166,10 @@ def __init__(self, dataloader: Iterable, device: Optional[torch.device] = None) self._dataloader = dataloader self._device = device - def __len__(self) -> int: - return len(self._dataloader) + def __len__(self) -> Union[int, float]: + if isinstance(self._dataloader, Sized): + return len(self._dataloader) + return float("inf") @property def device(self) -> Optional[torch.device]: diff --git a/pytorch_lightning/utilities/data.py b/pytorch_lightning/utilities/data.py index 4669cc2020b16..400148c77125b 100644 --- a/pytorch_lightning/utilities/data.py +++ b/pytorch_lightning/utilities/data.py @@ -46,7 +46,7 @@ def has_iterable_dataset(dataloader: DataLoader) -> bool: return hasattr(dataloader, "dataset") and isinstance(dataloader.dataset, IterableDataset) -def has_len(dataloader: DataLoader) -> bool: +def has_len(dataloader: Union[DataLoader, Iterable]) -> bool: """Checks if a given Dataloader has ``__len__`` method implemented i.e. if it is a finite dataloader or infinite dataloader. From 29bb0c9d7ba1d6173e09cf1627027ea19b65e1e8 Mon Sep 17 00:00:00 2001 From: tchaton Date: Mon, 1 Nov 2021 19:55:58 +0000 Subject: [PATCH 319/331] update --- pytorch_lightning/lite/wrappers.py | 4 +-- test.py | 52 ++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 test.py diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 782512885d20c..8b6f072c57adc 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -150,9 +150,7 @@ def _replace_dataloader_init_method() -> Generator: class _LiteDataLoader: - def __init__( - self, dataloader: Union[Iterable[Any], DataLoader[Any]], device: Optional[torch.device] = None - ) -> None: + def __init__(self, dataloader: Union[Iterable, DataLoader], device: Optional[torch.device] = None) -> None: """The LiteDataLoader is an extension of an Iterator. It would move the data to the device automatically if the device is specified. diff --git a/test.py b/test.py new file mode 100644 index 0000000000000..74acbdfc54e9b --- /dev/null +++ b/test.py @@ -0,0 +1,52 @@ +import torch +from torch.utils.data.dataloader import DataLoader + +from pytorch_lightning.loops.base import Loop + + +class TrainingBatchLoop(Loop): + def __init__(self, model, optimizer, dataloader): + super().__init__() + self.model = model + self.optimizer = optimizer + self.dataloader = dataloader + self.current_batch_idx = 0 + + @property + def done(self): + return self.current_batch_idx >= len(self.dataloader) + + def reset(self) -> None: + self.dataloader_iter = iter(self.dataloader) + + def advance(self, *args, **kwargs) -> None: + batch = next(self.dataloader_iter) + self.optimizer.zero_grad() + loss = self.model(batch) # , self.current_batch_idx) + loss.backward() + self.optimizer.step() + + +class EpochLoop(Loop): + def __init__(self, num_epochs, model, optimizer, dataloader): + super().__init__() + self.num_epochs = num_epochs + self.current_epoch = 0 + self.training_loop = TrainingBatchLoop(model, optimizer, dataloader) + + @property + def done(self): + return self.num_epochs < self.current_epoch + + def reset(self) -> None: + pass + + def advance(self, *args, **kwargs) -> None: + self.training_loop.run() + self.current_epoch += 1 + + +model = torch.nn.Linear(1, 1) +optimizer = torch.optim.SGD(model.parameters(), lr=0.1) +dataloader = DataLoader(torch.zeros((10,))) +EpochLoop(10, model, optimizer, dataloader).run() From 3b9496b5259e9ab080661d3b672b6eb5d9449c8e Mon Sep 17 00:00:00 2001 From: tchaton Date: Mon, 1 Nov 2021 20:04:02 +0000 Subject: [PATCH 320/331] remove test.py --- test.py | 52 ---------------------------------------------------- 1 file changed, 52 deletions(-) delete mode 100644 test.py diff --git a/test.py b/test.py deleted file mode 100644 index 74acbdfc54e9b..0000000000000 --- a/test.py +++ /dev/null @@ -1,52 +0,0 @@ -import torch -from torch.utils.data.dataloader import DataLoader - -from pytorch_lightning.loops.base import Loop - - -class TrainingBatchLoop(Loop): - def __init__(self, model, optimizer, dataloader): - super().__init__() - self.model = model - self.optimizer = optimizer - self.dataloader = dataloader - self.current_batch_idx = 0 - - @property - def done(self): - return self.current_batch_idx >= len(self.dataloader) - - def reset(self) -> None: - self.dataloader_iter = iter(self.dataloader) - - def advance(self, *args, **kwargs) -> None: - batch = next(self.dataloader_iter) - self.optimizer.zero_grad() - loss = self.model(batch) # , self.current_batch_idx) - loss.backward() - self.optimizer.step() - - -class EpochLoop(Loop): - def __init__(self, num_epochs, model, optimizer, dataloader): - super().__init__() - self.num_epochs = num_epochs - self.current_epoch = 0 - self.training_loop = TrainingBatchLoop(model, optimizer, dataloader) - - @property - def done(self): - return self.num_epochs < self.current_epoch - - def reset(self) -> None: - pass - - def advance(self, *args, **kwargs) -> None: - self.training_loop.run() - self.current_epoch += 1 - - -model = torch.nn.Linear(1, 1) -optimizer = torch.optim.SGD(model.parameters(), lr=0.1) -dataloader = DataLoader(torch.zeros((10,))) -EpochLoop(10, model, optimizer, dataloader).run() From 81390937369c37fd37d1fc7d930985141414d36f Mon Sep 17 00:00:00 2001 From: tchaton Date: Mon, 1 Nov 2021 20:36:27 +0000 Subject: [PATCH 321/331] update --- pl_examples/loop_examples/kfold.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/loop_examples/kfold.py b/pl_examples/loop_examples/kfold.py index 878ebc73054b7..bd14d42eb796f 100644 --- a/pl_examples/loop_examples/kfold.py +++ b/pl_examples/loop_examples/kfold.py @@ -27,7 +27,7 @@ from pl_examples import _DATASETS_PATH from pl_examples.basic_examples.mnist_datamodule import MNIST -from pl_examples.basic_examples.mnist_examples.image_classifier_4_lightning import ImageClassifier +from pl_examples.basic_examples.mnist_examples.image_classifier_4_lightning_module import ImageClassifier from pytorch_lightning import LightningDataModule, seed_everything, Trainer from pytorch_lightning.core.lightning import LightningModule from pytorch_lightning.loops.base import Loop From af13b284349782ed165b391f7561bcd5b239e63a Mon Sep 17 00:00:00 2001 From: tchaton Date: Mon, 1 Nov 2021 21:05:13 +0000 Subject: [PATCH 322/331] update --- .../basic_examples/mnist_examples/image_classifier_2_lite.py | 2 +- .../image_classifier_3_lite_to_lightning_module.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py index 213e1148f81c4..f47106fb142f1 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py @@ -70,7 +70,7 @@ def run(self, hparams): scheduler = StepLR(optimizer, step_size=1, gamma=hparams.gamma) # use torchmetrics instead of manually computing the accuracy - test_acc = Accuracy() + test_acc = Accuracy().to(self.device) # EPOCH LOOP for epoch in range(1, hparams.epochs + 1): diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning_module.py b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning_module.py index 12f318cea64dc..0d6925fc68c1a 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning_module.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_3_lite_to_lightning_module.py @@ -55,7 +55,8 @@ def run(self, hparams): self.prepare_data() train_loader, test_loader = self.setup_dataloaders(self.train_dataloader(), self.train_dataloader()) - self.test_acc = Accuracy() + + self.test_acc = Accuracy().to(self.device) # EPOCH LOOP for epoch in range(1, hparams.epochs + 1): From af1ad855e096e986fddffe441d89d983c5fab752 Mon Sep 17 00:00:00 2001 From: Thomas Chaton Date: Mon, 1 Nov 2021 17:08:49 -0400 Subject: [PATCH 323/331] update --- .../basic_examples/mnist_examples/image_classifier_2_lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py index f47106fb142f1..ae1b724df59b8 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py @@ -49,7 +49,7 @@ def run(self, hparams): seed_everything(hparams.seed) # instead of torch.manual_seed(...) transform = T.Compose([T.ToTensor(), T.Normalize((0.1307,), (0.3081,))]) - train_dataset = MNIST("./data", train=True, download=True, transform=transform) + train_dataset = MNIST("./data", train=True, download=self.is_global_zero, transform=transform) test_dataset = MNIST("./data", train=False, transform=transform) train_loader = torch.utils.data.DataLoader( train_dataset, From 389e535ca4ccf1260ce62804d8a6046f198c0d13 Mon Sep 17 00:00:00 2001 From: Thomas Chaton Date: Mon, 1 Nov 2021 17:12:16 -0400 Subject: [PATCH 324/331] update --- .../basic_examples/mnist_examples/image_classifier_2_lite.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py index ae1b724df59b8..0141f65adcdcd 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py @@ -49,7 +49,10 @@ def run(self, hparams): seed_everything(hparams.seed) # instead of torch.manual_seed(...) transform = T.Compose([T.ToTensor(), T.Normalize((0.1307,), (0.3081,))]) - train_dataset = MNIST("./data", train=True, download=self.is_global_zero, transform=transform) + if self.is_global_zero: + MNIST("./data", download=True) + self.barrier() + train_dataset = MNIST("./data", train=True, transform=transform) test_dataset = MNIST("./data", train=False, transform=transform) train_loader = torch.utils.data.DataLoader( train_dataset, From b4b63fb815f097a3c02f6b8dfef7d4f890fe6858 Mon Sep 17 00:00:00 2001 From: Thomas Chaton Date: Mon, 1 Nov 2021 17:13:35 -0400 Subject: [PATCH 325/331] update --- .../basic_examples/mnist_examples/image_classifier_2_lite.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py index 0141f65adcdcd..4240a9b7c4e08 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_2_lite.py @@ -49,6 +49,7 @@ def run(self, hparams): seed_everything(hparams.seed) # instead of torch.manual_seed(...) transform = T.Compose([T.ToTensor(), T.Normalize((0.1307,), (0.3081,))]) + # This is meant to ensure the data are download only by 1 process. if self.is_global_zero: MNIST("./data", download=True) self.barrier() From 3b82b5700c76798e172546ddcdf4fbafbdf5d13c Mon Sep 17 00:00:00 2001 From: Thomas Chaton Date: Mon, 1 Nov 2021 17:55:34 -0400 Subject: [PATCH 326/331] update --- pl_examples/loop_examples/mnist_lite.py | 12 +++++++----- pytorch_lightning/lite/wrappers.py | 4 ++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/pl_examples/loop_examples/mnist_lite.py b/pl_examples/loop_examples/mnist_lite.py index 13f68081bc1ba..9dd69a7adaa75 100644 --- a/pl_examples/loop_examples/mnist_lite.py +++ b/pl_examples/loop_examples/mnist_lite.py @@ -52,7 +52,7 @@ def advance(self, epoch) -> None: output = self.model(data) loss = F.nll_loss(output, target) self.lite.backward(loss) - self.optimizer.zero_grad() + self.optimizer.step() if (batch_idx == 0) or ((batch_idx + 1) % self.args.log_interval == 0): print( @@ -81,7 +81,7 @@ def __init__(self, lite, args, model, dataloader): self.model = model self.dataloader = dataloader self.dataloader_iter = None - self.accuracy = Accuracy() + self.accuracy = Accuracy().to(model.device) self.test_loss = 0 @property @@ -133,13 +133,15 @@ def advance(self, *args: Any, **kwargs: Any) -> None: raise StopIteration self.epoch += 1 - self.lite.val_acc.reset() class Lite(LightningLite): def run(self, hparams): transform = T.Compose([T.ToTensor(), T.Normalize((0.1307,), (0.3081,))]) - train_dataset = MNIST("./data", train=True, download=True, transform=transform) + if self.is_global_zero: + MNIST("./data", download=True) + self.barrier() + train_dataset = MNIST("./data", train=True, transform=transform) test_dataset = MNIST("./data", train=False, transform=transform) train_loader = torch.utils.data.DataLoader(train_dataset, hparams.batch_size) test_loader = torch.utils.data.DataLoader(test_dataset, hparams.test_batch_size) @@ -166,7 +168,7 @@ def run(self, hparams): parser.add_argument( "--test-batch-size", type=int, default=1000, metavar="N", help="input batch size for testing (default: 1000)" ) - parser.add_argument("--epochs", type=int, default=14, metavar="N", help="number of epochs to train (default: 14)") + parser.add_argument("--epochs", type=int, default=2, metavar="N", help="number of epochs to train (default: 14)") parser.add_argument("--lr", type=float, default=1.0, metavar="LR", help="learning rate (default: 1.0)") parser.add_argument("--gamma", type=float, default=0.7, metavar="M", help="Learning rate step gamma (default: 0.7)") parser.add_argument("--dry-run", action="store_true", default=False, help="quickly check a single pass") diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 8b6f072c57adc..5b69875263f99 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -78,6 +78,10 @@ def __init__(self, module: nn.Module, precision_plugin: PrecisionPlugin) -> None self._module = module self._precision_plugin = precision_plugin + @property + def device(self) -> torch.device: + return self._module.device + @property def module(self) -> nn.Module: return self._module From 7a88161266a265694cdb9ac9595c5eca64aa28d4 Mon Sep 17 00:00:00 2001 From: Thomas Chaton Date: Mon, 1 Nov 2021 18:06:39 -0400 Subject: [PATCH 327/331] update --- pl_examples/loop_examples/mnist_lite.py | 6 +++--- pytorch_lightning/lite/wrappers.py | 4 ---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/pl_examples/loop_examples/mnist_lite.py b/pl_examples/loop_examples/mnist_lite.py index 9dd69a7adaa75..4d59ef326f408 100644 --- a/pl_examples/loop_examples/mnist_lite.py +++ b/pl_examples/loop_examples/mnist_lite.py @@ -81,7 +81,7 @@ def __init__(self, lite, args, model, dataloader): self.model = model self.dataloader = dataloader self.dataloader_iter = None - self.accuracy = Accuracy().to(model.device) + self.accuracy = Accuracy().to(lite.device) self.test_loss = 0 @property @@ -156,7 +156,7 @@ def run(self, hparams): MainLoop(self, hparams, model, optimizer, scheduler, train_loader, test_loader).run() - if hparams.save_model: + if hparams.save_model and self.is_global_zero: self.save(model.state_dict(), "mnist_cnn.pt") @@ -185,4 +185,4 @@ def run(self, hparams): seed_everything(hparams.seed) - Lite(accelerator="auto", devices="auto").run(hparams) + Lite(accelerator="cpu", devices=1).run(hparams) diff --git a/pytorch_lightning/lite/wrappers.py b/pytorch_lightning/lite/wrappers.py index 5b69875263f99..8b6f072c57adc 100644 --- a/pytorch_lightning/lite/wrappers.py +++ b/pytorch_lightning/lite/wrappers.py @@ -78,10 +78,6 @@ def __init__(self, module: nn.Module, precision_plugin: PrecisionPlugin) -> None self._module = module self._precision_plugin = precision_plugin - @property - def device(self) -> torch.device: - return self._module.device - @property def module(self) -> nn.Module: return self._module From 506fc1933909ff715489b05f1d9dab1474fc0235 Mon Sep 17 00:00:00 2001 From: Thomas Chaton Date: Mon, 1 Nov 2021 18:47:50 -0400 Subject: [PATCH 328/331] update --- grid_generated_0.png | Bin 0 -> 6994 bytes grid_generated_1.png | Bin 0 -> 6994 bytes grid_ori_0.png | Bin 0 -> 1219 bytes pl_examples/run_examples.sh | 4 ++++ 4 files changed, 4 insertions(+) create mode 100644 grid_generated_0.png create mode 100644 grid_generated_1.png create mode 100644 grid_ori_0.png diff --git a/grid_generated_0.png b/grid_generated_0.png new file mode 100644 index 0000000000000000000000000000000000000000..09eb01ffee15ee95643f2930330ffffeaff11d5e GIT binary patch literal 6994 zcmW+*2|UyP8(+w==14*kOJX4;Im)q^BT14FLQC#rAtYy#BqT}WH_K0Q%$cM)la}U| z>nFL1Idk9qAOCq(UcLD2`97cLd0&e%Hqzrn3Ly~)1m`7voC*AE4S&$=Oz=04`_mEx z;^>h}xQnL#|I84_ZQEBjcC~qRc@@GQ$IHKbv5&P!d*8KvcP@SlW5%8OdEP)hHAg+K z%O&VzKw*bvn^Z~NwTnDBu6pAmQtbM7#h(|x#G>ubXekl5JOTq-^+HOkLcTm080tSO zm%qAPmEV0o2>W)I1YZqB%sZo_qmQ3F@#rfnDJ*eArmU!#)#)OYm}895)zcfO z4yktURXR_S_0|NtLVc|c2{s$w0d{Dz$kNZb(Nf)%o~Lxkw_0$rstQJ4@cq5 zD2Gw3EzQk;#>X`S*IFAIq)(m7uBZ@Z3d8UrB_%Wa%KfT?Hdh+ts#(IYo2zr&92`AG zcl2)GzKz8qqG1duGZc46-$s=`fn*M(g zadnyR-)q1(C{{Tc85I?k>Av#P;^M`%`4Kb^JBo$Y($Z2}n-1MD$PkPFalvm^QBe`6 zqjOGCacy&RuD`PU?c28>KD?|OIV~r5H0Jx$r%yve5hNo=MWK+jp~z>?kQlz4HYXOQ zeewEk9v-aWnlGYoVqB5s0n1W}20CfRG#V`?CdRcpKWKhfhd}t7s{Rm(L>gSaT=)IE zn^N!3pC-n}JmEDKmX;qZ^91BUt6xsjQ^L%E~$mWfm3|mX!tA*&&$11W`|Z{Lq(3$nJExBGFEZ77$hQ zM?`3qSe_=gtgfsCFHd&vl26^dc@rlV-!jh4&0Szy-S1M8V=5&t*k~v3ua=sU();7b z4@P^Yx3{5G3?zptH<1+lcot=jT1WG@Ce2hhZa_?PPVqCAZ85kJYyDNA>H+a}T zdJbra#j7YNd>xtRJ#+}BsK&0a#|?2Lu5Qu4;(Df0&;}#Z>XX}X9(HTlZW=A4y!?WO zhD||tc+DVOdv|9iV>v)Jsr*h|Y<_+|^aooU85xO13m!Rg$v#V??b|ntJTq}#>fMIdWo73{M%@DgS+6wc` z6&YSTWQ^fsVIsbH!xK&8VTZAN92vw{ghW(sgA+u7j*Z!De*?}9Xhl)P7XZrqD% zC%^VR=4kcy3QU#0l~wQW-`1|Ku3B1J*RNlP_TcgOd-v{{n|C9rMMXt$V1`MWF$Tq& z-p+7vZ~&}%WBA@$c6{F7He~z0p`oFlU#U$2Y;y%v^WT3K*RQvCbeO_XCMG5@ua1t6 zfq{Yd?wv_$s3Z)19Y}3~2S0J*#F!Vg#ttry9JJCyF)vsOQ0GG?((PwwXW3c4c6OeV zmUgX+J(il9N>EXQc|`Mku`ASz6IowhZ)#~tX>#CU=Mxm1n4Qfx#u$;#?`H~=y;b%N zU&PPP|Iu=0Y01aN#zs#s(xyN;r3offR#sLJ#Tr*P2W_3|%B!lXGRF99Z!Bx2@q}St zB_^(~t!0@?h1b}5dS1%|Js|NVtcw#p6ovEjoJ&hfgNFlL$g?WV zFDrxX7%01cpw%$L1YoYQ@wuVI=0tiNP|De}XIE$Y4!qFp;FC>CdHGU7b?*M$zYdpWi@`WKx5wu+hz%L=x%NjT<*iP5DShqJn~< zHFlRTUmgft+1lEgnRP8s zbzhOVuB-dR81uMx$W#hws4Z-hDMe5xKRS#XS|uD9m&NJ@ijL$mr7NI z@${SPhY5Hb5pmbivB6$JMR=CMxFi-o>ZOYPjxWN7!3w)}<+d?=;7wr_#CbX7iM^lP znZvN99)0Hnma?$u|Ehw@4J81e`}+D|RYL#3$--RYfBxu-^O|q`iZo0!$=ahtlaqxK4Gs$lmAyQrp`ihr z+FR;5@cG`z;&{s;US8l6z$rx2;^HFEsL4lv2pQ8d1reJAx z6>fM5bjsFRk!4=&vu8p_k8WBSzcV={%vC_4B+%_aZH3poF_d6u86Fu)NKKs`9Q-me z56=tgG?W80A2a_VtW9TUXF@{4RkB8jg$&%Xg^i7ev-35nM8pGMU;ZeZl9G}X9}?d8 zOSg=i+;z+=(gnY-c7^lv_bP2{ZETwD6{aR9*;$y#Eif+19>24)vUrdYfC{?lXOxu% zSePmSC~x21hla}r$?DjWdP$sDO(YJBUJ9s$H%>}24+uEVhdd@C(&eNG)4|EfnMa`j zMIDN!;l$>F8Q?IcutCsS4Gj%|bY>J-r2E0aJDwL0M&a;9Dj;5~a<(yuC9_ zvI`32up%NNsmaNfpdV~)$smD&1Qiq%3N2)WVs?h3ss#av(c z_~0PG?Nb~ch&(7roh$N=0ui>8C(l$0S_eCso_7;!EknR`hqV4i35}0-0TsdrmV~sULz$f zt+~B){q4WrD#GVUaP1(2e)abDxP3ClY|Yn+UPQM#l7m5~F3rp1oLGWA zw2(2&Z13sm0i^#kItp;j!gS5XCV1`dP*;~3Xy%=rU1{MHcC8Sti+ogwOioE>2E}Ff7a= zFlf6=iHfl7nKOH%3WvqU#`>JUC6TbcxoHkZ&eH_I+yGl4diwi=+uvUSE(ZfH;6;rUci& z8YvpAZ(v{|l~`L_d;Iuu;BseY=X2-I0j2=vHZ(Wykzb+#nn&&73W|kE9lTu>uHGKz zp4ZlP;7W=Tym11Z@5m9=vuEK6nb)#zmHQs%;{(ocDX|oZtHT$)GjVotak+AZGYVHm z7}5fbxv=oTra)9gBzR{_m7OIgH}_*{DVapt9*NVesHgz$e4UiU$*K+Q2Bqijo>y90 zO2g9|8!LPmys#W(vZ$~yii0EGAj3_m9G(Wquzaqnfll8yy$nLm-F?RP-2Mq~9`@1J zG;P2SB_)1jME%&pRCfV*CnW`i4;0GOFfDp}dwXYRM@2Z+P~yV*^9%lJf~b|hL$v^; zE6dB!?KZ|zMn=ZO#KhqNU}aFKm6h9V>3Vr)pr#)I<>ACmNl8skPoucGLl*xuf%ki$ zmRen{31)*g3T|d(WCXO^-Me>TF{Y-bfJNT6Iq^rIl#+4*VTafwe~03*n!yu#u^_tN zym>P=c9&?lGF4Fh>C-2GaHs-WxwO0t#v6p+$OtoQ_yaW&K4j3s=p|5$Z;4uIpxU57 zVDjqDS7Bjc!^6Yt>kmNPgITGsuZQ_RiefFXoMA9j)zpOf`TtB$%f#z%Y;NY6NvAZ; z&dgjfFv!WvYie#b!|(E9ekKYymc_fm$rlzKX!oTedJNf@|ZcyI8e?L7v z-JGg^fGO;=_3TV<>G=3K2aA@!zdt||flz2uz@`YE6$~I)OrqiUAp#cN`h@c+D;pay znvjst3Mk*gLU~0+mfU@h9zBwfkZ5!`EAJjG?+FV8n*ox&zh4B!`taey#r3VzR-YQ1 znm|;_$jX*csYhaTW%$9=0c)UGi!EflsA}Tk_Fi5l7{1wGC6w%JMv78 zhtvmh8erNzAsBO18IC`IjL3fxum29B6Fy{#Wu7>1RLeL-B`}kqx1|#el$DjikWWq` z#oq#O)9pb9?6;}JqJK0tx;Z(mt@L{KxcLSIlr=dp|7bnD2)%)>fGG6x@^W)?gX;!- ztKM3;?B}-%-vJUZh1n?z>FMcR!n>wzgvU{nPTA3mp|Vqs?HJN%6Uj+~x$qjVxo%iG$n2?!*ebM0z%`Uo*m zd3iaIG1Xc_Rka74p`|7A1v3*9LLoxyh_LV|K?TML07psbQ+oQy&h{p#o5_iZ|4Kc{ z5GE}xK^(vo20=+gWaQ@Bnp*IdV2m!*&|v?eAxRFB7Qg^d3lLD85qw^~{tEDvgXIxa z!Ws^CHQ%+^rWsguha;9ipCY`IV$os%vT)PKKZ=Hy4F2$UOnprfFmzZX(MRu)yuwwRn?k!^@X9$C7LjbG@yt zUxo-Uyr7*x8JJ`P)5F{xJ9Z4NAfOTq#%i5t2#aE|MJ`Bd7bFnwoal z5{D-yFbQ<)TesTV+e0=Ok|7}>MHZ;=noz&;+qJCXyr=l1GfbTA>;{*XKI)4rhcn%bui_0WPvI(D!r6onsP0r51#gGm_41kQ# z$}@vt=oxL8!C*{GIKfuBcFW2OPDB0yNIg9KsCEdJ7@{Oi4Gpdrq<{Y5L+Cq+vPpFxI+L;{DK4#4j}BoeUb6^TRf`r-+6X?r`Tt=>gq zO3HPqM0jv6E-rh8n`y=X1(0t-cLY%&X88E|yTF(YehK^a%M!L3Je=LF7YPa4>20AD z3JnV&vu*i`kJ$-$ZA-g-1 zlP+-7{i!aXF+j(ytw7kP^fo~O0l%^O!@NoWJDs3JZ*aet@2?m8i=>(>-Df6K>HO+M8Spk+y zk8)q6H#FRl7yMFsAlnoSh1JcQjKRUT-Bb6^lME%ieSMqj>vOWRYewc3g&?g7hNR;Q zTQcy7-R*tW(_kH!1A=bcIE;lT86Bg0R%kELB@u~Hv9bBNxhzjg!FUmgNA&dSfKecF zCT;lhR9j*iFwZ!v!~ zBvj^n|3z{7|0{##8c%4}?GH2fK0RUqWa$qzkl;4{9Ip&{ZH7k`(R=kDg# zh%aK29|E_=9}SV) z7d!bc9XqEnfD5eQ`@*m<==M=K$Z|$zXRGeiaRDrjjb*es?R~y+yY|WCeSp z4KkQmmuLIRg<>3Z79%A(+S|{90aPF=x&MOT9pXmdND$FawbFuj*2f|I1P+9h7_R5b z7pz^pBySW1`A}b$8O2%A($KK++c)?gCHvy6F&UlHa@$O`nQQj~e3dTlB91GuhAyH`_rYgnh)X&~xO^?UNB;wUvE)Di literal 0 HcmV?d00001 diff --git a/grid_generated_1.png b/grid_generated_1.png new file mode 100644 index 0000000000000000000000000000000000000000..09eb01ffee15ee95643f2930330ffffeaff11d5e GIT binary patch literal 6994 zcmW+*2|UyP8(+w==14*kOJX4;Im)q^BT14FLQC#rAtYy#BqT}WH_K0Q%$cM)la}U| z>nFL1Idk9qAOCq(UcLD2`97cLd0&e%Hqzrn3Ly~)1m`7voC*AE4S&$=Oz=04`_mEx z;^>h}xQnL#|I84_ZQEBjcC~qRc@@GQ$IHKbv5&P!d*8KvcP@SlW5%8OdEP)hHAg+K z%O&VzKw*bvn^Z~NwTnDBu6pAmQtbM7#h(|x#G>ubXekl5JOTq-^+HOkLcTm080tSO zm%qAPmEV0o2>W)I1YZqB%sZo_qmQ3F@#rfnDJ*eArmU!#)#)OYm}895)zcfO z4yktURXR_S_0|NtLVc|c2{s$w0d{Dz$kNZb(Nf)%o~Lxkw_0$rstQJ4@cq5 zD2Gw3EzQk;#>X`S*IFAIq)(m7uBZ@Z3d8UrB_%Wa%KfT?Hdh+ts#(IYo2zr&92`AG zcl2)GzKz8qqG1duGZc46-$s=`fn*M(g zadnyR-)q1(C{{Tc85I?k>Av#P;^M`%`4Kb^JBo$Y($Z2}n-1MD$PkPFalvm^QBe`6 zqjOGCacy&RuD`PU?c28>KD?|OIV~r5H0Jx$r%yve5hNo=MWK+jp~z>?kQlz4HYXOQ zeewEk9v-aWnlGYoVqB5s0n1W}20CfRG#V`?CdRcpKWKhfhd}t7s{Rm(L>gSaT=)IE zn^N!3pC-n}JmEDKmX;qZ^91BUt6xsjQ^L%E~$mWfm3|mX!tA*&&$11W`|Z{Lq(3$nJExBGFEZ77$hQ zM?`3qSe_=gtgfsCFHd&vl26^dc@rlV-!jh4&0Szy-S1M8V=5&t*k~v3ua=sU();7b z4@P^Yx3{5G3?zptH<1+lcot=jT1WG@Ce2hhZa_?PPVqCAZ85kJYyDNA>H+a}T zdJbra#j7YNd>xtRJ#+}BsK&0a#|?2Lu5Qu4;(Df0&;}#Z>XX}X9(HTlZW=A4y!?WO zhD||tc+DVOdv|9iV>v)Jsr*h|Y<_+|^aooU85xO13m!Rg$v#V??b|ntJTq}#>fMIdWo73{M%@DgS+6wc` z6&YSTWQ^fsVIsbH!xK&8VTZAN92vw{ghW(sgA+u7j*Z!De*?}9Xhl)P7XZrqD% zC%^VR=4kcy3QU#0l~wQW-`1|Ku3B1J*RNlP_TcgOd-v{{n|C9rMMXt$V1`MWF$Tq& z-p+7vZ~&}%WBA@$c6{F7He~z0p`oFlU#U$2Y;y%v^WT3K*RQvCbeO_XCMG5@ua1t6 zfq{Yd?wv_$s3Z)19Y}3~2S0J*#F!Vg#ttry9JJCyF)vsOQ0GG?((PwwXW3c4c6OeV zmUgX+J(il9N>EXQc|`Mku`ASz6IowhZ)#~tX>#CU=Mxm1n4Qfx#u$;#?`H~=y;b%N zU&PPP|Iu=0Y01aN#zs#s(xyN;r3offR#sLJ#Tr*P2W_3|%B!lXGRF99Z!Bx2@q}St zB_^(~t!0@?h1b}5dS1%|Js|NVtcw#p6ovEjoJ&hfgNFlL$g?WV zFDrxX7%01cpw%$L1YoYQ@wuVI=0tiNP|De}XIE$Y4!qFp;FC>CdHGU7b?*M$zYdpWi@`WKx5wu+hz%L=x%NjT<*iP5DShqJn~< zHFlRTUmgft+1lEgnRP8s zbzhOVuB-dR81uMx$W#hws4Z-hDMe5xKRS#XS|uD9m&NJ@ijL$mr7NI z@${SPhY5Hb5pmbivB6$JMR=CMxFi-o>ZOYPjxWN7!3w)}<+d?=;7wr_#CbX7iM^lP znZvN99)0Hnma?$u|Ehw@4J81e`}+D|RYL#3$--RYfBxu-^O|q`iZo0!$=ahtlaqxK4Gs$lmAyQrp`ihr z+FR;5@cG`z;&{s;US8l6z$rx2;^HFEsL4lv2pQ8d1reJAx z6>fM5bjsFRk!4=&vu8p_k8WBSzcV={%vC_4B+%_aZH3poF_d6u86Fu)NKKs`9Q-me z56=tgG?W80A2a_VtW9TUXF@{4RkB8jg$&%Xg^i7ev-35nM8pGMU;ZeZl9G}X9}?d8 zOSg=i+;z+=(gnY-c7^lv_bP2{ZETwD6{aR9*;$y#Eif+19>24)vUrdYfC{?lXOxu% zSePmSC~x21hla}r$?DjWdP$sDO(YJBUJ9s$H%>}24+uEVhdd@C(&eNG)4|EfnMa`j zMIDN!;l$>F8Q?IcutCsS4Gj%|bY>J-r2E0aJDwL0M&a;9Dj;5~a<(yuC9_ zvI`32up%NNsmaNfpdV~)$smD&1Qiq%3N2)WVs?h3ss#av(c z_~0PG?Nb~ch&(7roh$N=0ui>8C(l$0S_eCso_7;!EknR`hqV4i35}0-0TsdrmV~sULz$f zt+~B){q4WrD#GVUaP1(2e)abDxP3ClY|Yn+UPQM#l7m5~F3rp1oLGWA zw2(2&Z13sm0i^#kItp;j!gS5XCV1`dP*;~3Xy%=rU1{MHcC8Sti+ogwOioE>2E}Ff7a= zFlf6=iHfl7nKOH%3WvqU#`>JUC6TbcxoHkZ&eH_I+yGl4diwi=+uvUSE(ZfH;6;rUci& z8YvpAZ(v{|l~`L_d;Iuu;BseY=X2-I0j2=vHZ(Wykzb+#nn&&73W|kE9lTu>uHGKz zp4ZlP;7W=Tym11Z@5m9=vuEK6nb)#zmHQs%;{(ocDX|oZtHT$)GjVotak+AZGYVHm z7}5fbxv=oTra)9gBzR{_m7OIgH}_*{DVapt9*NVesHgz$e4UiU$*K+Q2Bqijo>y90 zO2g9|8!LPmys#W(vZ$~yii0EGAj3_m9G(Wquzaqnfll8yy$nLm-F?RP-2Mq~9`@1J zG;P2SB_)1jME%&pRCfV*CnW`i4;0GOFfDp}dwXYRM@2Z+P~yV*^9%lJf~b|hL$v^; zE6dB!?KZ|zMn=ZO#KhqNU}aFKm6h9V>3Vr)pr#)I<>ACmNl8skPoucGLl*xuf%ki$ zmRen{31)*g3T|d(WCXO^-Me>TF{Y-bfJNT6Iq^rIl#+4*VTafwe~03*n!yu#u^_tN zym>P=c9&?lGF4Fh>C-2GaHs-WxwO0t#v6p+$OtoQ_yaW&K4j3s=p|5$Z;4uIpxU57 zVDjqDS7Bjc!^6Yt>kmNPgITGsuZQ_RiefFXoMA9j)zpOf`TtB$%f#z%Y;NY6NvAZ; z&dgjfFv!WvYie#b!|(E9ekKYymc_fm$rlzKX!oTedJNf@|ZcyI8e?L7v z-JGg^fGO;=_3TV<>G=3K2aA@!zdt||flz2uz@`YE6$~I)OrqiUAp#cN`h@c+D;pay znvjst3Mk*gLU~0+mfU@h9zBwfkZ5!`EAJjG?+FV8n*ox&zh4B!`taey#r3VzR-YQ1 znm|;_$jX*csYhaTW%$9=0c)UGi!EflsA}Tk_Fi5l7{1wGC6w%JMv78 zhtvmh8erNzAsBO18IC`IjL3fxum29B6Fy{#Wu7>1RLeL-B`}kqx1|#el$DjikWWq` z#oq#O)9pb9?6;}JqJK0tx;Z(mt@L{KxcLSIlr=dp|7bnD2)%)>fGG6x@^W)?gX;!- ztKM3;?B}-%-vJUZh1n?z>FMcR!n>wzgvU{nPTA3mp|Vqs?HJN%6Uj+~x$qjVxo%iG$n2?!*ebM0z%`Uo*m zd3iaIG1Xc_Rka74p`|7A1v3*9LLoxyh_LV|K?TML07psbQ+oQy&h{p#o5_iZ|4Kc{ z5GE}xK^(vo20=+gWaQ@Bnp*IdV2m!*&|v?eAxRFB7Qg^d3lLD85qw^~{tEDvgXIxa z!Ws^CHQ%+^rWsguha;9ipCY`IV$os%vT)PKKZ=Hy4F2$UOnprfFmzZX(MRu)yuwwRn?k!^@X9$C7LjbG@yt zUxo-Uyr7*x8JJ`P)5F{xJ9Z4NAfOTq#%i5t2#aE|MJ`Bd7bFnwoal z5{D-yFbQ<)TesTV+e0=Ok|7}>MHZ;=noz&;+qJCXyr=l1GfbTA>;{*XKI)4rhcn%bui_0WPvI(D!r6onsP0r51#gGm_41kQ# z$}@vt=oxL8!C*{GIKfuBcFW2OPDB0yNIg9KsCEdJ7@{Oi4Gpdrq<{Y5L+Cq+vPpFxI+L;{DK4#4j}BoeUb6^TRf`r-+6X?r`Tt=>gq zO3HPqM0jv6E-rh8n`y=X1(0t-cLY%&X88E|yTF(YehK^a%M!L3Je=LF7YPa4>20AD z3JnV&vu*i`kJ$-$ZA-g-1 zlP+-7{i!aXF+j(ytw7kP^fo~O0l%^O!@NoWJDs3JZ*aet@2?m8i=>(>-Df6K>HO+M8Spk+y zk8)q6H#FRl7yMFsAlnoSh1JcQjKRUT-Bb6^lME%ieSMqj>vOWRYewc3g&?g7hNR;Q zTQcy7-R*tW(_kH!1A=bcIE;lT86Bg0R%kELB@u~Hv9bBNxhzjg!FUmgNA&dSfKecF zCT;lhR9j*iFwZ!v!~ zBvj^n|3z{7|0{##8c%4}?GH2fK0RUqWa$qzkl;4{9Ip&{ZH7k`(R=kDg# zh%aK29|E_=9}SV) z7d!bc9XqEnfD5eQ`@*m<==M=K$Z|$zXRGeiaRDrjjb*es?R~y+yY|WCeSp z4KkQmmuLIRg<>3Z79%A(+S|{90aPF=x&MOT9pXmdND$FawbFuj*2f|I1P+9h7_R5b z7pz^pBySW1`A}b$8O2%A($KK++c)?gCHvy6F&UlHa@$O`nQQj~e3dTlB91GuhAyH`_rYgnh)X&~xO^?UNB;wUvE)Di literal 0 HcmV?d00001 diff --git a/grid_ori_0.png b/grid_ori_0.png new file mode 100644 index 0000000000000000000000000000000000000000..497e4973b884cac667db79f054fb183d2a96819e GIT binary patch literal 1219 zcmV;!1U&nRP)AVGoz2@>Rw zVrFK>>-APwR|i_y*4E~5I9@+ua&l6sRDOrs-vNiiF&d2kfb;Y7fC`(Mn!uOXY___( zy6t9Qd{0>{R(W~(;Nal% z^Yg^S1a5mo_zPcIYHI5C_BNGDeSCZ@Dk@rATf<&Ur_*0vUS3~cKYSV;9lg4`DiVpN zrlu+?DzFIh`Fx2)l97?|S%Z*}5V>5AAV_z2cW-YmcB_GrnVAWPy4h?dQI^eSZ)|J; z0PJ>qPEHQSs*#Zq&l>;$k00-T9v&WI5yUfnI2=xYe?Po78jUyhl9uC z<>lqY#l}`UcR}xiBYGqv9YtW6J94JB?$xq9*<|WTIqCpWMt&o+1bj< zidZbx>2$cQ{yQ@>GjN#b_4=@|FdPbng@p!#;p*zjY&IVq9a$_E|An@`zMgc?5{bm) z@jN^{2!%q@#Su1}y}!Q?-4_=Z;}f~NyZd7I+1c62&(9~m0m4%`x z002eN($Z4gI$SPS{6=D8qVK*?D3r-$zQ}fWcgb*#Aa5fD<3bw%QBhG#OH0u1?CdPb z>VAZVhDxQ<>+5Tm%eB0`92*;pLxe`7<>lq|^z?Lhcel53)J(EVkKf005Org|TE{U|@K7xS*gwtJSWruYa~fqbMp62uQFP z5fO26a`Jt2w~C zXMTQOC=~wq@q+|&;o;$ni;FOkwY4?>Q+c4&*4DPPv~amxERC5==KcLW0N~)@0C$fN zxX?~dPZQMnh3^&a>FFswJsr1>TrPLJ-2i~&`1N+Ae>EG#Ve&JHOl zDFoG;nwoBJZ{eEf3wmg1C@n1wrV|zx#$vHpELK870!#z|(AwHcuwNJo03bIvm!K3v zqtUEZD|Dt%D2U3vp-?Es#>P+-g|o@G&8t+Zv$L~z2i&QtDdIgc7>v`?Q*uISXlQ`W z_V)JJY&LOo@2IJ%866#UI-UMwhx9}sCX>nQ^-85uqC)BE>F@|k;xPG;s;VlDM&lpH z+}s@TS>q3!jAU*={{svL1IAGP{ta&aBlPw4Aqc`|vuQNi*DdjXqy6L1>-E#q)0dZ* hpM41p5+n$5egYD7Ldw>)hDQJZ002ovPDHLkV1kjyQ?md7 literal 0 HcmV?d00001 diff --git a/pl_examples/run_examples.sh b/pl_examples/run_examples.sh index 321d39ce674fa..979ff40dfed74 100755 --- a/pl_examples/run_examples.sh +++ b/pl_examples/run_examples.sh @@ -9,6 +9,8 @@ args=" --trainer.limit_val_batches=2 --trainer.limit_test_batches=2 --trainer.limit_predict_batches=2 + --optimizer=Adam + --lr_scheduler=ConstantLR " python "${dir_path}/basic_examples/backbone_image_classifier.py" ${args} "$@" @@ -28,6 +30,8 @@ args=" --trainer.limit_val_batches=2 --trainer.limit_test_batches=2 --trainer.limit_predict_batches=2 + --optimizer=Adam + --lr_scheduler=ConstantLR " python "${dir_path}/basic_examples/mnist_examples/image_classifier_4_lightning_module.py" ${args} "$@" From 81ea11113828f92cd4d5f24db616c01b33c41416 Mon Sep 17 00:00:00 2001 From: Thomas Chaton Date: Mon, 1 Nov 2021 18:49:42 -0400 Subject: [PATCH 329/331] update --- grid_generated_0.png | Bin 6994 -> 0 bytes grid_generated_1.png | Bin 6994 -> 0 bytes grid_ori_0.png | Bin 1219 -> 0 bytes 3 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 grid_generated_0.png delete mode 100644 grid_generated_1.png delete mode 100644 grid_ori_0.png diff --git a/grid_generated_0.png b/grid_generated_0.png deleted file mode 100644 index 09eb01ffee15ee95643f2930330ffffeaff11d5e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6994 zcmW+*2|UyP8(+w==14*kOJX4;Im)q^BT14FLQC#rAtYy#BqT}WH_K0Q%$cM)la}U| z>nFL1Idk9qAOCq(UcLD2`97cLd0&e%Hqzrn3Ly~)1m`7voC*AE4S&$=Oz=04`_mEx z;^>h}xQnL#|I84_ZQEBjcC~qRc@@GQ$IHKbv5&P!d*8KvcP@SlW5%8OdEP)hHAg+K z%O&VzKw*bvn^Z~NwTnDBu6pAmQtbM7#h(|x#G>ubXekl5JOTq-^+HOkLcTm080tSO zm%qAPmEV0o2>W)I1YZqB%sZo_qmQ3F@#rfnDJ*eArmU!#)#)OYm}895)zcfO z4yktURXR_S_0|NtLVc|c2{s$w0d{Dz$kNZb(Nf)%o~Lxkw_0$rstQJ4@cq5 zD2Gw3EzQk;#>X`S*IFAIq)(m7uBZ@Z3d8UrB_%Wa%KfT?Hdh+ts#(IYo2zr&92`AG zcl2)GzKz8qqG1duGZc46-$s=`fn*M(g zadnyR-)q1(C{{Tc85I?k>Av#P;^M`%`4Kb^JBo$Y($Z2}n-1MD$PkPFalvm^QBe`6 zqjOGCacy&RuD`PU?c28>KD?|OIV~r5H0Jx$r%yve5hNo=MWK+jp~z>?kQlz4HYXOQ zeewEk9v-aWnlGYoVqB5s0n1W}20CfRG#V`?CdRcpKWKhfhd}t7s{Rm(L>gSaT=)IE zn^N!3pC-n}JmEDKmX;qZ^91BUt6xsjQ^L%E~$mWfm3|mX!tA*&&$11W`|Z{Lq(3$nJExBGFEZ77$hQ zM?`3qSe_=gtgfsCFHd&vl26^dc@rlV-!jh4&0Szy-S1M8V=5&t*k~v3ua=sU();7b z4@P^Yx3{5G3?zptH<1+lcot=jT1WG@Ce2hhZa_?PPVqCAZ85kJYyDNA>H+a}T zdJbra#j7YNd>xtRJ#+}BsK&0a#|?2Lu5Qu4;(Df0&;}#Z>XX}X9(HTlZW=A4y!?WO zhD||tc+DVOdv|9iV>v)Jsr*h|Y<_+|^aooU85xO13m!Rg$v#V??b|ntJTq}#>fMIdWo73{M%@DgS+6wc` z6&YSTWQ^fsVIsbH!xK&8VTZAN92vw{ghW(sgA+u7j*Z!De*?}9Xhl)P7XZrqD% zC%^VR=4kcy3QU#0l~wQW-`1|Ku3B1J*RNlP_TcgOd-v{{n|C9rMMXt$V1`MWF$Tq& z-p+7vZ~&}%WBA@$c6{F7He~z0p`oFlU#U$2Y;y%v^WT3K*RQvCbeO_XCMG5@ua1t6 zfq{Yd?wv_$s3Z)19Y}3~2S0J*#F!Vg#ttry9JJCyF)vsOQ0GG?((PwwXW3c4c6OeV zmUgX+J(il9N>EXQc|`Mku`ASz6IowhZ)#~tX>#CU=Mxm1n4Qfx#u$;#?`H~=y;b%N zU&PPP|Iu=0Y01aN#zs#s(xyN;r3offR#sLJ#Tr*P2W_3|%B!lXGRF99Z!Bx2@q}St zB_^(~t!0@?h1b}5dS1%|Js|NVtcw#p6ovEjoJ&hfgNFlL$g?WV zFDrxX7%01cpw%$L1YoYQ@wuVI=0tiNP|De}XIE$Y4!qFp;FC>CdHGU7b?*M$zYdpWi@`WKx5wu+hz%L=x%NjT<*iP5DShqJn~< zHFlRTUmgft+1lEgnRP8s zbzhOVuB-dR81uMx$W#hws4Z-hDMe5xKRS#XS|uD9m&NJ@ijL$mr7NI z@${SPhY5Hb5pmbivB6$JMR=CMxFi-o>ZOYPjxWN7!3w)}<+d?=;7wr_#CbX7iM^lP znZvN99)0Hnma?$u|Ehw@4J81e`}+D|RYL#3$--RYfBxu-^O|q`iZo0!$=ahtlaqxK4Gs$lmAyQrp`ihr z+FR;5@cG`z;&{s;US8l6z$rx2;^HFEsL4lv2pQ8d1reJAx z6>fM5bjsFRk!4=&vu8p_k8WBSzcV={%vC_4B+%_aZH3poF_d6u86Fu)NKKs`9Q-me z56=tgG?W80A2a_VtW9TUXF@{4RkB8jg$&%Xg^i7ev-35nM8pGMU;ZeZl9G}X9}?d8 zOSg=i+;z+=(gnY-c7^lv_bP2{ZETwD6{aR9*;$y#Eif+19>24)vUrdYfC{?lXOxu% zSePmSC~x21hla}r$?DjWdP$sDO(YJBUJ9s$H%>}24+uEVhdd@C(&eNG)4|EfnMa`j zMIDN!;l$>F8Q?IcutCsS4Gj%|bY>J-r2E0aJDwL0M&a;9Dj;5~a<(yuC9_ zvI`32up%NNsmaNfpdV~)$smD&1Qiq%3N2)WVs?h3ss#av(c z_~0PG?Nb~ch&(7roh$N=0ui>8C(l$0S_eCso_7;!EknR`hqV4i35}0-0TsdrmV~sULz$f zt+~B){q4WrD#GVUaP1(2e)abDxP3ClY|Yn+UPQM#l7m5~F3rp1oLGWA zw2(2&Z13sm0i^#kItp;j!gS5XCV1`dP*;~3Xy%=rU1{MHcC8Sti+ogwOioE>2E}Ff7a= zFlf6=iHfl7nKOH%3WvqU#`>JUC6TbcxoHkZ&eH_I+yGl4diwi=+uvUSE(ZfH;6;rUci& z8YvpAZ(v{|l~`L_d;Iuu;BseY=X2-I0j2=vHZ(Wykzb+#nn&&73W|kE9lTu>uHGKz zp4ZlP;7W=Tym11Z@5m9=vuEK6nb)#zmHQs%;{(ocDX|oZtHT$)GjVotak+AZGYVHm z7}5fbxv=oTra)9gBzR{_m7OIgH}_*{DVapt9*NVesHgz$e4UiU$*K+Q2Bqijo>y90 zO2g9|8!LPmys#W(vZ$~yii0EGAj3_m9G(Wquzaqnfll8yy$nLm-F?RP-2Mq~9`@1J zG;P2SB_)1jME%&pRCfV*CnW`i4;0GOFfDp}dwXYRM@2Z+P~yV*^9%lJf~b|hL$v^; zE6dB!?KZ|zMn=ZO#KhqNU}aFKm6h9V>3Vr)pr#)I<>ACmNl8skPoucGLl*xuf%ki$ zmRen{31)*g3T|d(WCXO^-Me>TF{Y-bfJNT6Iq^rIl#+4*VTafwe~03*n!yu#u^_tN zym>P=c9&?lGF4Fh>C-2GaHs-WxwO0t#v6p+$OtoQ_yaW&K4j3s=p|5$Z;4uIpxU57 zVDjqDS7Bjc!^6Yt>kmNPgITGsuZQ_RiefFXoMA9j)zpOf`TtB$%f#z%Y;NY6NvAZ; z&dgjfFv!WvYie#b!|(E9ekKYymc_fm$rlzKX!oTedJNf@|ZcyI8e?L7v z-JGg^fGO;=_3TV<>G=3K2aA@!zdt||flz2uz@`YE6$~I)OrqiUAp#cN`h@c+D;pay znvjst3Mk*gLU~0+mfU@h9zBwfkZ5!`EAJjG?+FV8n*ox&zh4B!`taey#r3VzR-YQ1 znm|;_$jX*csYhaTW%$9=0c)UGi!EflsA}Tk_Fi5l7{1wGC6w%JMv78 zhtvmh8erNzAsBO18IC`IjL3fxum29B6Fy{#Wu7>1RLeL-B`}kqx1|#el$DjikWWq` z#oq#O)9pb9?6;}JqJK0tx;Z(mt@L{KxcLSIlr=dp|7bnD2)%)>fGG6x@^W)?gX;!- ztKM3;?B}-%-vJUZh1n?z>FMcR!n>wzgvU{nPTA3mp|Vqs?HJN%6Uj+~x$qjVxo%iG$n2?!*ebM0z%`Uo*m zd3iaIG1Xc_Rka74p`|7A1v3*9LLoxyh_LV|K?TML07psbQ+oQy&h{p#o5_iZ|4Kc{ z5GE}xK^(vo20=+gWaQ@Bnp*IdV2m!*&|v?eAxRFB7Qg^d3lLD85qw^~{tEDvgXIxa z!Ws^CHQ%+^rWsguha;9ipCY`IV$os%vT)PKKZ=Hy4F2$UOnprfFmzZX(MRu)yuwwRn?k!^@X9$C7LjbG@yt zUxo-Uyr7*x8JJ`P)5F{xJ9Z4NAfOTq#%i5t2#aE|MJ`Bd7bFnwoal z5{D-yFbQ<)TesTV+e0=Ok|7}>MHZ;=noz&;+qJCXyr=l1GfbTA>;{*XKI)4rhcn%bui_0WPvI(D!r6onsP0r51#gGm_41kQ# z$}@vt=oxL8!C*{GIKfuBcFW2OPDB0yNIg9KsCEdJ7@{Oi4Gpdrq<{Y5L+Cq+vPpFxI+L;{DK4#4j}BoeUb6^TRf`r-+6X?r`Tt=>gq zO3HPqM0jv6E-rh8n`y=X1(0t-cLY%&X88E|yTF(YehK^a%M!L3Je=LF7YPa4>20AD z3JnV&vu*i`kJ$-$ZA-g-1 zlP+-7{i!aXF+j(ytw7kP^fo~O0l%^O!@NoWJDs3JZ*aet@2?m8i=>(>-Df6K>HO+M8Spk+y zk8)q6H#FRl7yMFsAlnoSh1JcQjKRUT-Bb6^lME%ieSMqj>vOWRYewc3g&?g7hNR;Q zTQcy7-R*tW(_kH!1A=bcIE;lT86Bg0R%kELB@u~Hv9bBNxhzjg!FUmgNA&dSfKecF zCT;lhR9j*iFwZ!v!~ zBvj^n|3z{7|0{##8c%4}?GH2fK0RUqWa$qzkl;4{9Ip&{ZH7k`(R=kDg# zh%aK29|E_=9}SV) z7d!bc9XqEnfD5eQ`@*m<==M=K$Z|$zXRGeiaRDrjjb*es?R~y+yY|WCeSp z4KkQmmuLIRg<>3Z79%A(+S|{90aPF=x&MOT9pXmdND$FawbFuj*2f|I1P+9h7_R5b z7pz^pBySW1`A}b$8O2%A($KK++c)?gCHvy6F&UlHa@$O`nQQj~e3dTlB91GuhAyH`_rYgnh)X&~xO^?UNB;wUvE)Di diff --git a/grid_generated_1.png b/grid_generated_1.png deleted file mode 100644 index 09eb01ffee15ee95643f2930330ffffeaff11d5e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6994 zcmW+*2|UyP8(+w==14*kOJX4;Im)q^BT14FLQC#rAtYy#BqT}WH_K0Q%$cM)la}U| z>nFL1Idk9qAOCq(UcLD2`97cLd0&e%Hqzrn3Ly~)1m`7voC*AE4S&$=Oz=04`_mEx z;^>h}xQnL#|I84_ZQEBjcC~qRc@@GQ$IHKbv5&P!d*8KvcP@SlW5%8OdEP)hHAg+K z%O&VzKw*bvn^Z~NwTnDBu6pAmQtbM7#h(|x#G>ubXekl5JOTq-^+HOkLcTm080tSO zm%qAPmEV0o2>W)I1YZqB%sZo_qmQ3F@#rfnDJ*eArmU!#)#)OYm}895)zcfO z4yktURXR_S_0|NtLVc|c2{s$w0d{Dz$kNZb(Nf)%o~Lxkw_0$rstQJ4@cq5 zD2Gw3EzQk;#>X`S*IFAIq)(m7uBZ@Z3d8UrB_%Wa%KfT?Hdh+ts#(IYo2zr&92`AG zcl2)GzKz8qqG1duGZc46-$s=`fn*M(g zadnyR-)q1(C{{Tc85I?k>Av#P;^M`%`4Kb^JBo$Y($Z2}n-1MD$PkPFalvm^QBe`6 zqjOGCacy&RuD`PU?c28>KD?|OIV~r5H0Jx$r%yve5hNo=MWK+jp~z>?kQlz4HYXOQ zeewEk9v-aWnlGYoVqB5s0n1W}20CfRG#V`?CdRcpKWKhfhd}t7s{Rm(L>gSaT=)IE zn^N!3pC-n}JmEDKmX;qZ^91BUt6xsjQ^L%E~$mWfm3|mX!tA*&&$11W`|Z{Lq(3$nJExBGFEZ77$hQ zM?`3qSe_=gtgfsCFHd&vl26^dc@rlV-!jh4&0Szy-S1M8V=5&t*k~v3ua=sU();7b z4@P^Yx3{5G3?zptH<1+lcot=jT1WG@Ce2hhZa_?PPVqCAZ85kJYyDNA>H+a}T zdJbra#j7YNd>xtRJ#+}BsK&0a#|?2Lu5Qu4;(Df0&;}#Z>XX}X9(HTlZW=A4y!?WO zhD||tc+DVOdv|9iV>v)Jsr*h|Y<_+|^aooU85xO13m!Rg$v#V??b|ntJTq}#>fMIdWo73{M%@DgS+6wc` z6&YSTWQ^fsVIsbH!xK&8VTZAN92vw{ghW(sgA+u7j*Z!De*?}9Xhl)P7XZrqD% zC%^VR=4kcy3QU#0l~wQW-`1|Ku3B1J*RNlP_TcgOd-v{{n|C9rMMXt$V1`MWF$Tq& z-p+7vZ~&}%WBA@$c6{F7He~z0p`oFlU#U$2Y;y%v^WT3K*RQvCbeO_XCMG5@ua1t6 zfq{Yd?wv_$s3Z)19Y}3~2S0J*#F!Vg#ttry9JJCyF)vsOQ0GG?((PwwXW3c4c6OeV zmUgX+J(il9N>EXQc|`Mku`ASz6IowhZ)#~tX>#CU=Mxm1n4Qfx#u$;#?`H~=y;b%N zU&PPP|Iu=0Y01aN#zs#s(xyN;r3offR#sLJ#Tr*P2W_3|%B!lXGRF99Z!Bx2@q}St zB_^(~t!0@?h1b}5dS1%|Js|NVtcw#p6ovEjoJ&hfgNFlL$g?WV zFDrxX7%01cpw%$L1YoYQ@wuVI=0tiNP|De}XIE$Y4!qFp;FC>CdHGU7b?*M$zYdpWi@`WKx5wu+hz%L=x%NjT<*iP5DShqJn~< zHFlRTUmgft+1lEgnRP8s zbzhOVuB-dR81uMx$W#hws4Z-hDMe5xKRS#XS|uD9m&NJ@ijL$mr7NI z@${SPhY5Hb5pmbivB6$JMR=CMxFi-o>ZOYPjxWN7!3w)}<+d?=;7wr_#CbX7iM^lP znZvN99)0Hnma?$u|Ehw@4J81e`}+D|RYL#3$--RYfBxu-^O|q`iZo0!$=ahtlaqxK4Gs$lmAyQrp`ihr z+FR;5@cG`z;&{s;US8l6z$rx2;^HFEsL4lv2pQ8d1reJAx z6>fM5bjsFRk!4=&vu8p_k8WBSzcV={%vC_4B+%_aZH3poF_d6u86Fu)NKKs`9Q-me z56=tgG?W80A2a_VtW9TUXF@{4RkB8jg$&%Xg^i7ev-35nM8pGMU;ZeZl9G}X9}?d8 zOSg=i+;z+=(gnY-c7^lv_bP2{ZETwD6{aR9*;$y#Eif+19>24)vUrdYfC{?lXOxu% zSePmSC~x21hla}r$?DjWdP$sDO(YJBUJ9s$H%>}24+uEVhdd@C(&eNG)4|EfnMa`j zMIDN!;l$>F8Q?IcutCsS4Gj%|bY>J-r2E0aJDwL0M&a;9Dj;5~a<(yuC9_ zvI`32up%NNsmaNfpdV~)$smD&1Qiq%3N2)WVs?h3ss#av(c z_~0PG?Nb~ch&(7roh$N=0ui>8C(l$0S_eCso_7;!EknR`hqV4i35}0-0TsdrmV~sULz$f zt+~B){q4WrD#GVUaP1(2e)abDxP3ClY|Yn+UPQM#l7m5~F3rp1oLGWA zw2(2&Z13sm0i^#kItp;j!gS5XCV1`dP*;~3Xy%=rU1{MHcC8Sti+ogwOioE>2E}Ff7a= zFlf6=iHfl7nKOH%3WvqU#`>JUC6TbcxoHkZ&eH_I+yGl4diwi=+uvUSE(ZfH;6;rUci& z8YvpAZ(v{|l~`L_d;Iuu;BseY=X2-I0j2=vHZ(Wykzb+#nn&&73W|kE9lTu>uHGKz zp4ZlP;7W=Tym11Z@5m9=vuEK6nb)#zmHQs%;{(ocDX|oZtHT$)GjVotak+AZGYVHm z7}5fbxv=oTra)9gBzR{_m7OIgH}_*{DVapt9*NVesHgz$e4UiU$*K+Q2Bqijo>y90 zO2g9|8!LPmys#W(vZ$~yii0EGAj3_m9G(Wquzaqnfll8yy$nLm-F?RP-2Mq~9`@1J zG;P2SB_)1jME%&pRCfV*CnW`i4;0GOFfDp}dwXYRM@2Z+P~yV*^9%lJf~b|hL$v^; zE6dB!?KZ|zMn=ZO#KhqNU}aFKm6h9V>3Vr)pr#)I<>ACmNl8skPoucGLl*xuf%ki$ zmRen{31)*g3T|d(WCXO^-Me>TF{Y-bfJNT6Iq^rIl#+4*VTafwe~03*n!yu#u^_tN zym>P=c9&?lGF4Fh>C-2GaHs-WxwO0t#v6p+$OtoQ_yaW&K4j3s=p|5$Z;4uIpxU57 zVDjqDS7Bjc!^6Yt>kmNPgITGsuZQ_RiefFXoMA9j)zpOf`TtB$%f#z%Y;NY6NvAZ; z&dgjfFv!WvYie#b!|(E9ekKYymc_fm$rlzKX!oTedJNf@|ZcyI8e?L7v z-JGg^fGO;=_3TV<>G=3K2aA@!zdt||flz2uz@`YE6$~I)OrqiUAp#cN`h@c+D;pay znvjst3Mk*gLU~0+mfU@h9zBwfkZ5!`EAJjG?+FV8n*ox&zh4B!`taey#r3VzR-YQ1 znm|;_$jX*csYhaTW%$9=0c)UGi!EflsA}Tk_Fi5l7{1wGC6w%JMv78 zhtvmh8erNzAsBO18IC`IjL3fxum29B6Fy{#Wu7>1RLeL-B`}kqx1|#el$DjikWWq` z#oq#O)9pb9?6;}JqJK0tx;Z(mt@L{KxcLSIlr=dp|7bnD2)%)>fGG6x@^W)?gX;!- ztKM3;?B}-%-vJUZh1n?z>FMcR!n>wzgvU{nPTA3mp|Vqs?HJN%6Uj+~x$qjVxo%iG$n2?!*ebM0z%`Uo*m zd3iaIG1Xc_Rka74p`|7A1v3*9LLoxyh_LV|K?TML07psbQ+oQy&h{p#o5_iZ|4Kc{ z5GE}xK^(vo20=+gWaQ@Bnp*IdV2m!*&|v?eAxRFB7Qg^d3lLD85qw^~{tEDvgXIxa z!Ws^CHQ%+^rWsguha;9ipCY`IV$os%vT)PKKZ=Hy4F2$UOnprfFmzZX(MRu)yuwwRn?k!^@X9$C7LjbG@yt zUxo-Uyr7*x8JJ`P)5F{xJ9Z4NAfOTq#%i5t2#aE|MJ`Bd7bFnwoal z5{D-yFbQ<)TesTV+e0=Ok|7}>MHZ;=noz&;+qJCXyr=l1GfbTA>;{*XKI)4rhcn%bui_0WPvI(D!r6onsP0r51#gGm_41kQ# z$}@vt=oxL8!C*{GIKfuBcFW2OPDB0yNIg9KsCEdJ7@{Oi4Gpdrq<{Y5L+Cq+vPpFxI+L;{DK4#4j}BoeUb6^TRf`r-+6X?r`Tt=>gq zO3HPqM0jv6E-rh8n`y=X1(0t-cLY%&X88E|yTF(YehK^a%M!L3Je=LF7YPa4>20AD z3JnV&vu*i`kJ$-$ZA-g-1 zlP+-7{i!aXF+j(ytw7kP^fo~O0l%^O!@NoWJDs3JZ*aet@2?m8i=>(>-Df6K>HO+M8Spk+y zk8)q6H#FRl7yMFsAlnoSh1JcQjKRUT-Bb6^lME%ieSMqj>vOWRYewc3g&?g7hNR;Q zTQcy7-R*tW(_kH!1A=bcIE;lT86Bg0R%kELB@u~Hv9bBNxhzjg!FUmgNA&dSfKecF zCT;lhR9j*iFwZ!v!~ zBvj^n|3z{7|0{##8c%4}?GH2fK0RUqWa$qzkl;4{9Ip&{ZH7k`(R=kDg# zh%aK29|E_=9}SV) z7d!bc9XqEnfD5eQ`@*m<==M=K$Z|$zXRGeiaRDrjjb*es?R~y+yY|WCeSp z4KkQmmuLIRg<>3Z79%A(+S|{90aPF=x&MOT9pXmdND$FawbFuj*2f|I1P+9h7_R5b z7pz^pBySW1`A}b$8O2%A($KK++c)?gCHvy6F&UlHa@$O`nQQj~e3dTlB91GuhAyH`_rYgnh)X&~xO^?UNB;wUvE)Di diff --git a/grid_ori_0.png b/grid_ori_0.png deleted file mode 100644 index 497e4973b884cac667db79f054fb183d2a96819e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1219 zcmV;!1U&nRP)AVGoz2@>Rw zVrFK>>-APwR|i_y*4E~5I9@+ua&l6sRDOrs-vNiiF&d2kfb;Y7fC`(Mn!uOXY___( zy6t9Qd{0>{R(W~(;Nal% z^Yg^S1a5mo_zPcIYHI5C_BNGDeSCZ@Dk@rATf<&Ur_*0vUS3~cKYSV;9lg4`DiVpN zrlu+?DzFIh`Fx2)l97?|S%Z*}5V>5AAV_z2cW-YmcB_GrnVAWPy4h?dQI^eSZ)|J; z0PJ>qPEHQSs*#Zq&l>;$k00-T9v&WI5yUfnI2=xYe?Po78jUyhl9uC z<>lqY#l}`UcR}xiBYGqv9YtW6J94JB?$xq9*<|WTIqCpWMt&o+1bj< zidZbx>2$cQ{yQ@>GjN#b_4=@|FdPbng@p!#;p*zjY&IVq9a$_E|An@`zMgc?5{bm) z@jN^{2!%q@#Su1}y}!Q?-4_=Z;}f~NyZd7I+1c62&(9~m0m4%`x z002eN($Z4gI$SPS{6=D8qVK*?D3r-$zQ}fWcgb*#Aa5fD<3bw%QBhG#OH0u1?CdPb z>VAZVhDxQ<>+5Tm%eB0`92*;pLxe`7<>lq|^z?Lhcel53)J(EVkKf005Org|TE{U|@K7xS*gwtJSWruYa~fqbMp62uQFP z5fO26a`Jt2w~C zXMTQOC=~wq@q+|&;o;$ni;FOkwY4?>Q+c4&*4DPPv~amxERC5==KcLW0N~)@0C$fN zxX?~dPZQMnh3^&a>FFswJsr1>TrPLJ-2i~&`1N+Ae>EG#Ve&JHOl zDFoG;nwoBJZ{eEf3wmg1C@n1wrV|zx#$vHpELK870!#z|(AwHcuwNJo03bIvm!K3v zqtUEZD|Dt%D2U3vp-?Es#>P+-g|o@G&8t+Zv$L~z2i&QtDdIgc7>v`?Q*uISXlQ`W z_V)JJY&LOo@2IJ%866#UI-UMwhx9}sCX>nQ^-85uqC)BE>F@|k;xPG;s;VlDM&lpH z+}s@TS>q3!jAU*={{svL1IAGP{ta&aBlPw4Aqc`|vuQNi*DdjXqy6L1>-E#q)0dZ* hpM41p5+n$5egYD7Ldw>)hDQJZ002ovPDHLkV1kjyQ?md7 From 5209601afe7a4a596d23bafb5294f2e942f0d587 Mon Sep 17 00:00:00 2001 From: tchaton Date: Mon, 1 Nov 2021 23:26:57 +0000 Subject: [PATCH 330/331] update --- pl_examples/run_examples.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/pl_examples/run_examples.sh b/pl_examples/run_examples.sh index 979ff40dfed74..4a15c3367d35f 100755 --- a/pl_examples/run_examples.sh +++ b/pl_examples/run_examples.sh @@ -10,7 +10,6 @@ args=" --trainer.limit_test_batches=2 --trainer.limit_predict_batches=2 --optimizer=Adam - --lr_scheduler=ConstantLR " python "${dir_path}/basic_examples/backbone_image_classifier.py" ${args} "$@" @@ -31,7 +30,6 @@ args=" --trainer.limit_test_batches=2 --trainer.limit_predict_batches=2 --optimizer=Adam - --lr_scheduler=ConstantLR " python "${dir_path}/basic_examples/mnist_examples/image_classifier_4_lightning_module.py" ${args} "$@" From 33b87588a60e0bf689b2d90b850e452774e76cd8 Mon Sep 17 00:00:00 2001 From: tchaton Date: Tue, 2 Nov 2021 07:26:21 +0000 Subject: [PATCH 331/331] update --- .../mnist_examples/image_classifier_4_lightning_module.py | 4 ---- .../image_classifier_5_lightning_datamodule.py | 6 ------ 2 files changed, 10 deletions(-) diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning_module.py b/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning_module.py index 6ca06f151520b..cb67d3446c51a 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning_module.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_4_lightning_module.py @@ -18,7 +18,6 @@ import torch import torchvision.transforms as T from torch.nn import functional as F -from torchmetrics.classification import Accuracy from pl_examples import cli_lightning_logo from pl_examples.basic_examples.mnist_datamodule import MNIST @@ -32,7 +31,6 @@ def __init__(self, model=None, lr=1.0, gamma=0.7, batch_size=32): super().__init__() self.save_hyperparameters() self.model = model or Net() - self.test_acc = Accuracy() def forward(self, x): return self.model(x) @@ -47,8 +45,6 @@ def test_step(self, batch, batch_idx): x, y = batch logits = self.forward(x) loss = F.nll_loss(logits, y.long()) - self.test_acc(logits, y.long()) - self.log("test_acc", self.test_acc) return loss def configure_optimizers(self): diff --git a/pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py b/pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py index a6970c30ecbf4..4020d101ccab6 100644 --- a/pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py +++ b/pl_examples/basic_examples/mnist_examples/image_classifier_5_lightning_datamodule.py @@ -18,7 +18,6 @@ import torch import torchvision.transforms as T from torch.nn import functional as F -from torchmetrics.classification import Accuracy from pl_examples import cli_lightning_logo from pl_examples.basic_examples.mnist_datamodule import MNIST @@ -32,7 +31,6 @@ def __init__(self, model, lr=1.0, gamma=0.7, batch_size=32): super().__init__() self.save_hyperparameters() self.model = model or Net() - self.test_acc = Accuracy() def forward(self, x): return self.model(x) @@ -47,12 +45,8 @@ def test_step(self, batch, batch_idx): x, y = batch logits = self.forward(x) loss = F.nll_loss(logits, y.long()) - self.test_acc(logits, y.long()) return loss - def test_epoch_end(self, *_) -> None: - self.log("test_acc", self.test_acc.compute()) - def configure_optimizers(self): optimizer = torch.optim.Adadelta(self.model.parameters(), lr=self.hparams.lr) return [optimizer], [torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=self.hparams.gamma)]