Feature: wrapper for callbacks

bin/tests/check_dl_core_ignore_metric_callback.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+
+# Cause the script to exit if a single command fails
+set -eo pipefail -v
+
+
+################################  global variables  ################################
+rm -rf ./tests/logs ./tests/output.txt
+
+EXPDIR=./tests/_tests_contrib_dl_callbacks
+LOGDIR=./tests/logs/_tests_contrib_dl_callbacks
+CHECKPOINTS=${LOGDIR}/checkpoints
+LOGFILE=${CHECKPOINTS}/_metrics.json
+EXP_OUTPUT=./tests/output.txt
+
+
+function check_file_existence {
+    # $1 - path to file
+    if [[ ! -f "$1" ]]
+    then
+        echo "There is no '$1'!"
+        exit 1
+    fi
+}
+
+
+function check_num_files {
+    # $1 - ls directory
+    # $2 - expected count
+    NFILES=$( ls $1 | wc -l )
+    if [[ $NFILES -ne $2 ]]
+    then
+        echo "Different number of files in '$1' - "`
+              `"expected $2 but actual number is $NFILES!"
+        exit 1
+    fi
+}
+
+
+function check_checkpoints {
+    # $1 - file prefix
+    # $2 - expected count
+    check_num_files "${1}.pth" $2
+    check_num_files "${1}_full.pth" $2
+}
+
+
+function check_line_counts {
+    # $1 file
+    # $2 pattern
+    # $3 expected count
+    ACTUAL_COUNT=$( grep -c "$2" $1 || true )  # '|| true' for handling pipefail
+    if [ $ACTUAL_COUNT -ne $3 ]
+    then
+        echo "Different number of lines in file '$1' - "`
+             `"expected $3 (should match '$2') but actual number is $ACTUAL_COUNT!"
+        exit 1
+    fi
+}
+
+################################  pipeline 00  ################################
+# setup: run validation once in 3 epoch
+LOG_MSG='pipeline 00'
+echo ${LOG_MSG}
+
+LOGDIR=./tests/logs/_tests_dl_callbacks
+CHECKPOINTS=${LOGDIR}/checkpoints
+LOGFILE=${CHECKPOINTS}/_metrics.json
+EXP_OUTPUT=./tests/output.txt
+
+PYTHONPATH=./examples:./catalyst:${PYTHONPATH} \
+  python3 -c "
+import torch
+from torch.utils.data import DataLoader, TensorDataset
+from catalyst.dl import (
+    SupervisedRunner, Callback, CallbackOrder,
+    IgnoreMetricCallback, AccuracyCallback,
+)
+
+# experiment_setup
+logdir = '${LOGDIR}'
+num_epochs = 10
+
+# data
+num_samples, num_features = int(1e4), int(1e1)
+X = torch.rand(num_samples, num_features)
+y = torch.randint(0, 5, size=[num_samples])
+dataset = TensorDataset(X, y)
+loader = DataLoader(dataset, batch_size=32, num_workers=1)
+loaders = {
+    'train': loader,
+    'valid': loader,
+}
+
+# model, criterion, optimizer, scheduler
+model = torch.nn.Linear(num_features, 5)
+criterion = torch.nn.CrossEntropyLoss()
+optimizer = torch.optim.Adam(model.parameters())
+runner = SupervisedRunner()
+
+# first stage
+runner.train(
+    model=model,
+    criterion=criterion,
+    optimizer=optimizer,
+    loaders=loaders,
+    logdir=logdir,
+    num_epochs=num_epochs,
+    verbose=True,
+    main_metric='accuracy01',
+    callbacks=[
+        AccuracyCallback(accuracy_args=[1, 3, 5]),
+        IgnoreMetricCallback(valid=['_criterion'])
+    ]
+)
+" > ${EXP_OUTPUT}
+
+cat ${EXP_OUTPUT}
+# check_line_counts ${EXP_OUTPUT} "(train):" 10
+# check_line_counts ${EXP_OUTPUT} "(valid):" 3
+# check_line_counts ${EXP_OUTPUT} ".*/train\.[[:digit:]]\.pth" 1
+
+check_file_existence ${LOGFILE}
+cat ${LOGFILE}
+echo ${LOG_MSG}
+
+check_checkpoints "${CHECKPOINTS}/best" 1
+check_checkpoints "${CHECKPOINTS}/last" 1
+check_checkpoints "${CHECKPOINTS}/train\.[[:digit:]]" 1
+check_num_files ${CHECKPOINTS} 7   # 3x2 checkpoints + metrics.json
+
+rm -rf ${LOGDIR} ${EXP_OUTPUT}

catalyst/contrib/dl/callbacks/__init__.py

@@ -5,6 +5,7 @@
 
 from .cutmix_callback import CutmixCallback
 from .gradnorm_logger import GradNormLogger
+from .ignore_metric_callback import IgnoreMetricCallback
 from .knn_metric import KNNMetricCallback
 from .periodic_loader_callback import PeriodicLoaderCallback
 from .perplexity_metric import PerplexityMetricCallback

catalyst/contrib/dl/callbacks/ignore_metric_callback.py

@@ -0,0 +1,78 @@
+from catalyst.core.callback import Callback, CallbackOrder
+from catalyst.core.runner import IRunner
+
+
+class IgnoreMetricCallback(Callback):
+    """
+    Ignore metric callbacks for specified loaders.
+    """
+
+    def __init__(self, **kwargs):
+        """
+
+        Args:
+            kwargs: loader and callback names to ignore
+        """
+        super().__init__(order=CallbackOrder.External)
+        # contains pointers to callbacks
+        self.callbacks = {}
+        self.loader_ignore_list = {}
+        for loader, ignore_list in kwargs.items():
+            if not isinstance(ignore_list, (str, list, tuple)):
+                raise TypeError(
+                    "Expected ignore list object is str/List[str]/Tuple[str] "
+                    f"but got {type(ignore_list)}"
+                )
+            if isinstance(ignore_list, str):
+                to_ignore = [ignore_list]
+            else:
+                to_ignore = [
+                    str(callback_name) for callback_name in ignore_list
+                ]
+            self.loader_ignore_list[loader] = to_ignore
+
+    def on_stage_start(self, runner: IRunner) -> None:
+        """Get information about callbacks used in a stage.
+
+        Args:
+            runner (IRunner): current runner
+        """
+        for name, callback in runner.callbacks.items():
+            self.callbacks[name] = callback
+
+    def _is_correct_loader(
+        self, loader: str, name: str, callback: Callback
+    ) -> bool:
+        """
+        Check if callback should be used with loader.
+
+        Args:
+            loader (str): loader name
+            name (str): callback name
+            callback (Callback): callback object
+
+        Returns:
+            True if callback should be used with passed loader otherwise False
+        """
+        ignore_list = self.loader_ignore_list.get(loader) or []
+        in_ignore_list = name in ignore_list
+        is_metric = callback.order in (
+            CallbackOrder.Metric,
+            CallbackOrder.MetricAggregation,
+        )
+        return not (in_ignore_list and is_metric)
+
+    def on_loader_start(self, runner: IRunner) -> None:
+        """
+        Construct list of callbacks for current loader.
+
+        Args:
+            runner (IRunner): current runner
+        """
+        loader = runner.loader_name
+        filtered_loader_callbacks = {
+            name: callback
+            for name, callback in self.callbacks.items()
+            if self._is_correct_loader(loader, name, callback)
+        }
+        runner.callbacks = filtered_loader_callbacks