diff --git a/CHANGELOG.md b/CHANGELOG.md index ec6772ea3407f..83f32bbba29ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,10 +26,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - `stat_scores` metric now calculates stat scores over all classes and gains new parameters, in line with the new `StatScores` metric ([#4839](https://github.com/PyTorchLightning/pytorch-lightning/pull/4839)) + ### Deprecated - `stat_scores_multiple_classes` is deprecated in favor of `stat_scores` ([#4839](https://github.com/PyTorchLightning/pytorch-lightning/pull/4839)) + ### Removed - Removed deprecated checkpoint argument `filepath` ([#5321](https://github.com/PyTorchLightning/pytorch-lightning/pull/5321)) @@ -38,9 +40,13 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Removed deprecated `Fbeta`, `f1_score` and `fbeta_score` metrics ([#5322](https://github.com/PyTorchLightning/pytorch-lightning/pull/5322)) +- Removed deprecated `TrainResult` ([#5323](https://github.com/PyTorchLightning/pytorch-lightning/pull/5323)) + + ### Fixed + ## [1.1.0] - 2020-12-09 ### Added diff --git a/pytorch_lightning/core/step_result.py b/pytorch_lightning/core/step_result.py index 7101ec17c4bbc..f42ed53f64fbe 100644 --- a/pytorch_lightning/core/step_result.py +++ b/pytorch_lightning/core/step_result.py @@ -681,157 +681,6 @@ def collate_tensors(items: Union[List, Tuple]) -> Union[Tensor, List, Tuple]: return items -class TrainResult(Result): - def __init__( - self, - minimize: Optional[Tensor] = None, - early_stop_on: Optional[Tensor] = None, - checkpoint_on: Optional[Union[Tensor, bool]] = None, - hiddens: Optional[Tensor] = None, - ): - """ - Tracks internal metrics aggregations - - Args: - minimize: Metric currently being minimized. - early_stop_on: Metric to early stop on. - Should be a one element tensor if combined with default - :class:`~pytorch_lightning.callbacks.early_stopping.EarlyStopping`. - If this result is returned by - :meth:`~pytorch_lightning.core.lightning.LightningModule.training_step`, - the specified value will be averaged across all steps. - checkpoint_on: Metric to checkpoint on. - Should be a one element tensor if combined with default checkpoint callback. - If this result is returned by - :meth:`~pytorch_lightning.core.lightning.LightningModule.training_step`, - the specified value will be averaged across all steps. - hiddens: - """ - - super().__init__(minimize, early_stop_on, checkpoint_on, hiddens) - - def log( - self, - name, - value, - prog_bar: bool = False, - logger: bool = True, - on_step: bool = True, - on_epoch: bool = False, - reduce_fx: Callable = torch.mean, - tbptt_reduce_fx: Callable = torch.mean, - tbptt_pad_token: int = 0, - enable_graph: bool = False, - sync_dist: bool = False, - sync_dist_op: Union[Any, str] = 'mean', - sync_dist_group: Optional[Any] = None, - ): - """ - Log a key, value - - Example:: - - result.log('train_loss', loss) - - # defaults used - result.log( - name, - value, - on_step=True, - on_epoch=False, - logger=True, - prog_bar=False, - reduce_fx=torch.mean, - enable_graph=False - ) - - - Args: - name: key name - value: value name - prog_bar: if True logs to the progress base - logger: if True logs to the logger - on_step: if True logs the output of validation_step or test_step - on_epoch: if True, logs the output of the training loop aggregated - reduce_fx: Torch.mean by default - tbptt_reduce_fx: function to reduce on truncated back prop - tbptt_pad_token: token to use for padding - enable_graph: if True, will not auto detach the graph - sync_dist: if True, reduces the metric across GPUs/TPUs - sync_dist_op: the op to sync across - sync_dist_group: the ddp group - """ - super().log( - name=name, - value=value, - prog_bar=prog_bar, - logger=logger, - on_step=on_step, - on_epoch=on_epoch, - reduce_fx=reduce_fx, - enable_graph=enable_graph, - sync_dist=sync_dist, - sync_dist_group=sync_dist_group, - sync_dist_op=sync_dist_op, - tbptt_pad_token=tbptt_pad_token, - tbptt_reduce_fx=tbptt_reduce_fx, - ) - - def log_dict( - self, - dictionary: dict, - prog_bar: bool = False, - logger: bool = True, - on_step: bool = False, - on_epoch: bool = True, - reduce_fx: Callable = torch.mean, - tbptt_reduce_fx: Callable = torch.mean, - tbptt_pad_token: int = 0, - enable_graph: bool = False, - sync_dist: bool = False, - sync_dist_op: Union[Any, str] = 'mean', - sync_dist_group: Optional[Any] = None, - ): - """ - Log a dictonary of values at once - - Example:: - - values = {'loss': loss, 'acc': acc, ..., 'metric_n': metric_n} - result.log_dict(values) - - Args: - dictionary: key value pairs (str, tensors) - prog_bar: if True logs to the progress base - logger: if True logs to the logger - on_step: if True logs the output of validation_step or test_step - on_epoch: if True, logs the output of the training loop aggregated - reduce_fx: Torch.mean by default - tbptt_reduce_fx: function to reduce on truncated back prop - tbptt_pad_token: token to use for padding - enable_graph: if True, will not auto detach the graph - sync_dist: if True, reduces the metric across GPUs/TPUs - sync_dist_op: the op to sync across - sync_dist_group: the ddp group: - """ - for k, v in dictionary.items(): - self.log( - name=k, - value=v, - prog_bar=prog_bar, - logger=logger, - on_step=on_step, - on_epoch=on_epoch, - reduce_fx=reduce_fx, - enable_graph=enable_graph, - sync_dist=sync_dist, - sync_dist_group=sync_dist_group, - sync_dist_op=sync_dist_op, - tbptt_pad_token=tbptt_pad_token, - tbptt_reduce_fx=tbptt_reduce_fx, - ) - - class EvalResult(Result): def __init__( self, diff --git a/pytorch_lightning/overrides/data_parallel.py b/pytorch_lightning/overrides/data_parallel.py index 855f026701c7d..7b4b22eb8a234 100644 --- a/pytorch_lightning/overrides/data_parallel.py +++ b/pytorch_lightning/overrides/data_parallel.py @@ -104,11 +104,7 @@ def __gather_structured_result(self, outputs): outputs = self.gather(outputs) - # pass minimize to constructor for TrainResult - if 'minimize' in outputs: - result = original_class(outputs['minimize']) - else: - result = original_class() + result = original_class() result.update(outputs) result['meta'] = meta diff --git a/tests/base/model_train_steps.py b/tests/base/model_train_steps.py index 6bbdc5357a7a7..773d8cd2ad80f 100644 --- a/tests/base/model_train_steps.py +++ b/tests/base/model_train_steps.py @@ -17,8 +17,6 @@ import torch -from pytorch_lightning.core.step_result import TrainResult - class TrainingStepVariations(ABC): """ @@ -28,27 +26,27 @@ class TrainingStepVariations(ABC): test_step_inf_loss = float('inf') def training_step(self, batch, batch_idx, optimizer_idx=None): + """Lightning calls this inside the training loop""" self.training_step_called = True - """Lightning calls this inside the training loop""" # forward pass x, y = batch x = x.view(x.size(0), -1) y_hat = self(x) # calculate loss - loss_val = self.loss(y, y_hat) - log_val = loss_val + loss_train = self.loss(y, y_hat) + log_train = loss_train # alternate between tensors and scalars for "log" and "progress_bar" if batch_idx % 2 == 0: - log_val = log_val.item() + log_train = log_train.item() output = OrderedDict( { - 'loss': loss_val, - 'progress_bar': {'some_val': log_val * log_val}, - 'log': {'train_some_val': log_val * log_val}, + 'loss': loss_train, + 'progress_bar': {'some_val': log_train * log_train}, + 'log': {'train_some_val': log_train * log_train}, } ) return output @@ -62,24 +60,22 @@ def training_step__inf_loss(self, batch, batch_idx, optimizer_idx=None): output /= 0 return output - def training_step_result_obj_dp(self, batch, batch_idx, optimizer_idx=None): + def training_step__result_obj_dp(self, batch, batch_idx, optimizer_idx=None): + # forward pass x, y = batch x = x.view(x.size(0), -1) y_hat = self(x.to(self.device)) # calculate loss - loss_val = self.loss(y.to(y_hat.device), y_hat) - log_val = loss_val + loss_train = self.loss(y.to(y_hat.device), y_hat) + log_train = loss_train # alternate between tensors and scalars for "log" and "progress_bar" if batch_idx % 2 == 0: - log_val = log_val.item() - - result = TrainResult(loss_val) - result.log('some_val', log_val * log_val, prog_bar=True, logger=False) - result.log('train_some_val', log_val * log_val) + log_train = log_train.item() - self.training_step_called = True + self.log('some_val', log_train * log_train, prog_bar=True, logger=False) + self.log('train_some_val', log_train * log_train) - return result + return loss_train diff --git a/tests/base/model_valid_steps.py b/tests/base/model_valid_steps.py index 92a8cbeee58e0..af242572723ee 100644 --- a/tests/base/model_valid_steps.py +++ b/tests/base/model_valid_steps.py @@ -47,7 +47,7 @@ def validation_step(self, batch, batch_idx, *args, **kwargs): }) return output - def validation_step_result_obj_dp(self, batch, batch_idx, *args, **kwargs): + def validation_step__result_obj_dp(self, batch, batch_idx, *args, **kwargs): x, y = batch x = x.view(x.size(0), -1) y_hat = self(x.to(self.device)) diff --git a/tests/checkpointing/test_model_checkpoint.py b/tests/checkpointing/test_model_checkpoint.py index d4f2e863c7d21..27f484c63d87c 100644 --- a/tests/checkpointing/test_model_checkpoint.py +++ b/tests/checkpointing/test_model_checkpoint.py @@ -493,10 +493,9 @@ def training_step(self, batch, batch_idx): if batch_idx % 2 == 0: log_val = log_val.item() - result = pl.core.step_result.TrainResult(loss_val) - result.log('some_val', log_val * log_val, prog_bar=True, logger=False) - result.log('train_some_val', log_val * log_val) - return result + self.log('some_val', log_val * log_val, prog_bar=True, logger=False) + self.log('train_some_val', log_val * log_val) + return loss_val def validation_step(self, batch, batch_idx): y_hat = self(batch) diff --git a/tests/core/test_results.py b/tests/core/test_results.py index d9fd3a473a15e..1108ebf0e8460 100644 --- a/tests/core/test_results.py +++ b/tests/core/test_results.py @@ -19,7 +19,7 @@ import torch.distributed as dist import torch.multiprocessing as mp from pytorch_lightning import Trainer -from pytorch_lightning.core.step_result import Result, TrainResult, EvalResult +from pytorch_lightning.core.step_result import Result, EvalResult import tests.base.develop_utils as tutils from tests.base import EvalModelTemplate @@ -45,7 +45,7 @@ def _ddp_test_fn(rank, worldsize, result_cls: Result): assert res["test_tensor"].item() == dist.get_world_size(), "Result-Log does not work properly with DDP and Tensors" -@pytest.mark.parametrize("result_cls", [Result, TrainResult, EvalResult]) +@pytest.mark.parametrize("result_cls", [Result, EvalResult]) @pytest.mark.skipif(sys.platform == "win32", reason="DDP not available on windows") def test_result_reduce_ddp(result_cls): """Make sure result logging works with DDP""" diff --git a/tests/models/test_cpu.py b/tests/models/test_cpu.py index 2848ab2e74f3c..892077ccdb1be 100644 --- a/tests/models/test_cpu.py +++ b/tests/models/test_cpu.py @@ -22,7 +22,6 @@ import tests.base.develop_utils as tutils from pytorch_lightning import Trainer from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint -from pytorch_lightning.core.step_result import TrainResult from tests.base import EvalModelTemplate @@ -379,151 +378,3 @@ def train_dataloader(self): result = trainer.fit(model) assert result == 1, 'training failed to complete' - - -def test_tbptt_cpu_model_result(tmpdir): - """Test truncated back propagation through time works.""" - truncated_bptt_steps = 2 - sequence_size = 30 - batch_size = 30 - - x_seq = torch.rand(batch_size, sequence_size, 1) - y_seq_list = torch.rand(batch_size, sequence_size, 1).tolist() - - class MockSeq2SeqDataset(torch.utils.data.Dataset): - def __getitem__(self, i): - return x_seq, y_seq_list - - def __len__(self): - return 1 - - class BpttTestModel(EvalModelTemplate): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.test_hidden = None - - def training_step(self, batch, batch_idx, hiddens): - assert hiddens == self.test_hidden, "Hidden state not persistent between tbptt steps" - self.test_hidden = torch.rand(1) - - x_tensor, y_list = batch - assert x_tensor.shape[1] == truncated_bptt_steps, "tbptt split Tensor failed" - - y_tensor = torch.tensor(y_list, dtype=x_tensor.dtype) - assert y_tensor.shape[1] == truncated_bptt_steps, "tbptt split list failed" - - pred = self(x_tensor.view(batch_size, truncated_bptt_steps)) - loss_val = torch.nn.functional.mse_loss( - pred, y_tensor.view(batch_size, truncated_bptt_steps)) - - result = TrainResult(loss_val, hiddens=self.test_hidden) - return result - - def training_epoch_end(self, training_step_outputs): - result = training_step_outputs - assert isinstance(result, TrainResult) - assert result.minimize.size(1) == (sequence_size / truncated_bptt_steps) - - result.minimize = result.minimize.mean() - return result - - def train_dataloader(self): - return torch.utils.data.DataLoader( - dataset=MockSeq2SeqDataset(), - batch_size=batch_size, - shuffle=False, - sampler=None, - ) - - hparams = EvalModelTemplate.get_default_hparams() - hparams.update( - batch_size=batch_size, - in_features=truncated_bptt_steps, - hidden_dim=truncated_bptt_steps, - out_features=truncated_bptt_steps - ) - - model = BpttTestModel(**hparams) - model.example_input_array = torch.randn(5, truncated_bptt_steps) - - # fit model - trainer = Trainer( - default_root_dir=tmpdir, - max_epochs=1, - truncated_bptt_steps=truncated_bptt_steps, - limit_val_batches=0, - weights_summary=None, - ) - result = trainer.fit(model) - - assert result == 1, 'training failed to complete' - - -def test_tbptt_cpu_model_result_auto_reduce(tmpdir): - """Test truncated back propagation through time works.""" - truncated_bptt_steps = 2 - sequence_size = 30 - batch_size = 30 - - x_seq = torch.rand(batch_size, sequence_size, 1) - y_seq_list = torch.rand(batch_size, sequence_size, 1).tolist() - - class MockSeq2SeqDataset(torch.utils.data.Dataset): - def __getitem__(self, i): - return x_seq, y_seq_list - - def __len__(self): - return 1 - - class BpttTestModel(EvalModelTemplate): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.test_hidden = None - - def training_step(self, batch, batch_idx, hiddens): - assert hiddens == self.test_hidden, "Hidden state not persistent between tbptt steps" - self.test_hidden = torch.rand(1) - - x_tensor, y_list = batch - assert x_tensor.shape[1] == truncated_bptt_steps, "tbptt split Tensor failed" - - y_tensor = torch.tensor(y_list, dtype=x_tensor.dtype) - assert y_tensor.shape[1] == truncated_bptt_steps, "tbptt split list failed" - - pred = self(x_tensor.view(batch_size, truncated_bptt_steps)) - loss_val = torch.nn.functional.mse_loss( - pred, y_tensor.view(batch_size, truncated_bptt_steps)) - - result = TrainResult(loss_val, hiddens=self.test_hidden) - return result - - def train_dataloader(self): - return torch.utils.data.DataLoader( - dataset=MockSeq2SeqDataset(), - batch_size=batch_size, - shuffle=False, - sampler=None, - ) - - hparams = EvalModelTemplate.get_default_hparams() - hparams.update( - batch_size=batch_size, - in_features=truncated_bptt_steps, - hidden_dim=truncated_bptt_steps, - out_features=truncated_bptt_steps - ) - - model = BpttTestModel(**hparams) - model.example_input_array = torch.randn(5, truncated_bptt_steps) - - # fit model - trainer = Trainer( - default_root_dir=tmpdir, - max_epochs=1, - truncated_bptt_steps=truncated_bptt_steps, - limit_val_batches=0, - weights_summary=None, - ) - result = trainer.fit(model) - - assert result == 1, 'training failed to complete' diff --git a/tests/models/test_sync_batchnorm.py b/tests/models/test_sync_batchnorm.py index fd771c98635ab..03145b7ff9d78 100644 --- a/tests/models/test_sync_batchnorm.py +++ b/tests/models/test_sync_batchnorm.py @@ -17,7 +17,6 @@ import torch.nn.functional as F from pytorch_lightning import Trainer, seed_everything, LightningModule -from pytorch_lightning.core.step_result import TrainResult from pytorch_lightning.utilities import FLOAT16_EPSILON from tests.base.datamodules import MNISTDataModule from tests.base.develop_utils import set_random_master_port @@ -57,7 +56,7 @@ def training_step(self, batch, batch_idx): y_hat, _ = self(x, batch_idx) loss = F.cross_entropy(y_hat, y) - return TrainResult(loss) + return loss def configure_optimizers(self): return torch.optim.Adam(self.linear.parameters(), lr=0.02) diff --git a/tests/trainer/test_correct_freq_accumulation.py b/tests/trainer/test_correct_freq_accumulation.py index 2753a919b7728..1dcaed8e783a0 100644 --- a/tests/trainer/test_correct_freq_accumulation.py +++ b/tests/trainer/test_correct_freq_accumulation.py @@ -30,10 +30,10 @@ def test_training_step_scalar(tmpdir): model = EvalModelTemplate() model.validation_step = None model.test_step = None - model.training_step = model.training_step_result_obj_dp + model.training_step = model.training_step__result_obj_dp model.training_step_end = None model.training_epoch_end = None - model.validation_step = model.validation_step_result_obj_dp + model.validation_step = model.validation_step__result_obj_dp model.validation_step_end = None model.validation_epoch_end = None model.test_dataloader = None