diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml
index d9ffdb7f5e..e568ba9e15 100644
--- a/.github/workflows/cron.yml
+++ b/.github/workflows/cron.yml
@@ -53,6 +53,38 @@ jobs:
         fail_ci_if_error: false
         file: ./coverage.xml
 
+  cron-pt-image:
+    if: github.repository == 'Project-MONAI/MONAI'
+    container:
+      image: nvcr.io/nvidia/pytorch:20.12-py3  # testing with the latest pytorch base image
+      options: "--gpus all"
+    runs-on: [self-hosted, linux, x64, common]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install the dependencies
+      run: |
+        which python
+        python -m pip install --upgrade pip wheel
+        python -m pip install -r requirements-dev.txt
+        python -m pip list
+    - name: Run tests report coverage
+      run: |
+        export LAUNCH_DELAY=$[ $RANDOM % 16 * 60 ]
+        echo "Sleep $LAUNCH_DELAY"
+        sleep $LAUNCH_DELAY
+        nvidia-smi
+        export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
+        echo $CUDA_VISIBLE_DEVICES
+        python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
+        python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
+        BUILD_MONAI=1 ./runtests.sh --coverage
+        coverage xml
+    - name: Upload coverage
+      uses: codecov/codecov-action@v1
+      with:
+        fail_ci_if_error: false
+        file: ./coverage.xml
+
   cron-docker:
     if: github.repository == 'Project-MONAI/MONAI'
     container:
@@ -61,7 +93,8 @@ jobs:
     runs-on: [self-hosted, linux, x64, common]
     steps:
     - name: Run tests report coverage
-      # The docker image process has done the compilation.  BUILD_MONAI=1 may not be necessary.
+      # The docker image process has done the compilation.
+      # BUILD_MONAI=1 is necessary for triggering the USE_COMPILED flag.
       run: |
         cd /opt/monai
         nvidia-smi
diff --git a/Dockerfile b/Dockerfile
index a600f9de84..47976b97b1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -9,7 +9,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG PYTORCH_IMAGE=nvcr.io/nvidia/pytorch:20.10-py3
+ARG PYTORCH_IMAGE=nvcr.io/nvidia/pytorch:20.12-py3
 
 FROM ${PYTORCH_IMAGE}
 
@@ -21,7 +21,8 @@ WORKDIR /opt/monai
 COPY requirements.txt requirements-min.txt requirements-dev.txt /tmp/
 RUN cp /tmp/requirements.txt /tmp/req.bak \
   && awk '!/torch/' /tmp/requirements.txt > /tmp/tmp && mv /tmp/tmp /tmp/requirements.txt \
-  && python -m pip install --no-cache-dir --use-feature=2020-resolver -r /tmp/requirements-dev.txt
+  && python -m pip install --upgrade --no-cache-dir pip \
+  && python -m pip install --no-cache-dir -r /tmp/requirements-dev.txt
 
 # compile ext and remove temp files
 # TODO: remark for issue [revise the dockerfile #1276](https://github.com/Project-MONAI/MONAI/issues/1276)
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 5614e2bf40..d046bc53cf 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -2,7 +2,7 @@
 torch>=1.5
 pytorch-ignite==0.4.2
 numpy>=1.17
-itk
+itk>=5.0
 nibabel
 parameterized
 scikit-image>=0.14.2
diff --git a/docs/source/apps.rst b/docs/source/apps.rst
index f2afd93836..b8c8b4d341 100644
--- a/docs/source/apps.rst
+++ b/docs/source/apps.rst
@@ -28,3 +28,27 @@ Applications
 .. autofunction:: extractall
 
 .. autofunction:: download_and_extract
+
+`Deepgrow`
+----------
+
+.. automodule:: monai.apps.deepgrow.dataset
+.. autofunction:: create_dataset
+
+.. automodule:: monai.apps.deepgrow.interaction
+.. autoclass:: Interaction
+    :members:
+
+.. automodule:: monai.apps.deepgrow.transforms
+.. autoclass:: AddInitialSeedPointd
+    :members:
+.. autoclass:: AddGuidanceSignald
+    :members:
+.. autoclass:: AddRandomGuidanced
+    :members:
+.. autoclass:: SpatialCropForegroundd
+    :members:
+.. autoclass:: FindDiscrepancyRegionsd
+    :members:
+.. autoclass:: FindAllValidSlicesd
+    :members:
diff --git a/docs/source/handlers.rst b/docs/source/handlers.rst
index 81d28fb4ac..a629b28b27 100644
--- a/docs/source/handlers.rst
+++ b/docs/source/handlers.rst
@@ -85,6 +85,9 @@ Training stats handler
 
 Tensorboard handlers
 --------------------
+.. autoclass:: TensorBoardHandler
+    :members:
+
 .. autoclass:: TensorBoardStatsHandler
     :members:
 
diff --git a/docs/source/losses.rst b/docs/source/losses.rst
index a6aa4d566d..5e19219fee 100644
--- a/docs/source/losses.rst
+++ b/docs/source/losses.rst
@@ -74,4 +74,12 @@ Registration Losses
 `GlobalMutualInformationLoss`
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 .. autoclass:: GlobalMutualInformationLoss
+    :members:
+
+Loss Wrappers
+--------------
+
+`MultiScaleLoss`
+~~~~~~~~~~~~~~~~~
+.. autoclass:: MultiScaleLoss
     :members:
\ No newline at end of file
diff --git a/docs/source/networks.rst b/docs/source/networks.rst
index 7c22964835..e0ac0f2d75 100644
--- a/docs/source/networks.rst
+++ b/docs/source/networks.rst
@@ -120,12 +120,12 @@ Blocks
 .. autoclass:: SubpixelUpSample
 
 `LocalNet DownSample Block`
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
 .. autoclass:: LocalNetDownSampleBlock
     :members:
 
 `LocalNet UpSample Block`
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~
 .. autoclass:: LocalNetUpSampleBlock
     :members:
 
@@ -135,10 +135,15 @@ Blocks
     :members:
 
 `Warp`
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~
 .. autoclass:: Warp
     :members:
 
+`DVF2DDF`
+~~~~~~~~~
+.. autoclass:: DVF2DDF
+    :members:
+
 Layers
 ------
 
@@ -201,7 +206,7 @@ Layers
 ~~~~~~~~~~~~~~~~
 .. autoclass:: GaussianFilter
     :members:
-    
+
 `BilateralFilter`
 ~~~~~~~~~~~~~~~~~
 .. autoclass:: BilateralFilter
diff --git a/docs/source/transforms.rst b/docs/source/transforms.rst
index 90d960a6b9..5b550f7885 100644
--- a/docs/source/transforms.rst
+++ b/docs/source/transforms.rst
@@ -237,6 +237,12 @@ IO
     :members:
     :special-members: __call__
 
+`SaveImage`
+"""""""""""
+.. autoclass:: SaveImage
+    :members:
+    :special-members: __call__
+
 Post-processing
 ^^^^^^^^^^^^^^^
 
@@ -702,6 +708,12 @@ IO (Dict)
     :members:
     :special-members: __call__
 
+`SaveImaged`
+""""""""""""
+.. autoclass:: SaveImaged
+    :members:
+    :special-members: __call__
+
 Post-processing (Dict)
 ^^^^^^^^^^^^^^^^^^^^^^
 
@@ -939,6 +951,12 @@ Utility (Dict)
     :members:
     :special-members: __call__
 
+`RandLambdad`
+"""""""""""""
+.. autoclass:: RandLambdad
+    :members:
+    :special-members: __call__
+
 `LabelToMaskd`
 """"""""""""""
 .. autoclass:: LabelToMaskd
diff --git a/monai/apps/datasets.py b/monai/apps/datasets.py
index d8fd815ce9..f0416b8c4f 100644
--- a/monai/apps/datasets.py
+++ b/monai/apps/datasets.py
@@ -94,7 +94,9 @@ def __init__(
         data = self._generate_data_list(dataset_dir)
         if transform == ():
             transform = LoadImaged("image")
-        super().__init__(data, transform, cache_num=cache_num, cache_rate=cache_rate, num_workers=num_workers)
+        CacheDataset.__init__(
+            self, data, transform, cache_num=cache_num, cache_rate=cache_rate, num_workers=num_workers
+        )
 
     def randomize(self, data: Optional[Any] = None) -> None:
         self.rann = self.R.random()
@@ -275,7 +277,9 @@ def __init__(
         self._properties = load_decathlon_properties(os.path.join(dataset_dir, "dataset.json"), property_keys)
         if transform == ():
             transform = LoadImaged(["image", "label"])
-        super().__init__(data, transform, cache_num=cache_num, cache_rate=cache_rate, num_workers=num_workers)
+        CacheDataset.__init__(
+            self, data, transform, cache_num=cache_num, cache_rate=cache_rate, num_workers=num_workers
+        )
 
     def get_indices(self) -> np.ndarray:
         """
diff --git a/monai/apps/deepgrow/__init__.py b/monai/apps/deepgrow/__init__.py
new file mode 100644
index 0000000000..14ae193634
--- /dev/null
+++ b/monai/apps/deepgrow/__init__.py
@@ -0,0 +1,10 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/monai/apps/deepgrow/dataset.py b/monai/apps/deepgrow/dataset.py
new file mode 100644
index 0000000000..45cfbde6ea
--- /dev/null
+++ b/monai/apps/deepgrow/dataset.py
@@ -0,0 +1,281 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+from typing import Dict, List
+
+import numpy as np
+
+from monai.transforms import AsChannelFirstd, Compose, LoadImaged, Orientationd, Spacingd
+from monai.utils import GridSampleMode
+
+
+def create_dataset(
+    datalist,
+    output_dir: str,
+    dimension,
+    pixdim,
+    image_key: str = "image",
+    label_key: str = "label",
+    base_dir=None,
+    limit: int = 0,
+    relative_path: bool = False,
+    transforms=None,
+) -> List[Dict]:
+    """
+    Utility to pre-process and create dataset list for Deepgrow training over on existing one.
+    The input data list is normally a list of images and labels (3D volume) that needs pre-processing
+    for Deepgrow training pipeline.
+
+    Args:
+        datalist: A list of data dictionary. Each entry should at least contain 'image_key': <image filename>.
+            For example, typical input data can be a list of dictionaries::
+
+                [{'image': <image filename>, 'label': <label filename>}]
+
+        output_dir: target directory to store the training data for Deepgrow Training
+        pixdim: output voxel spacing.
+        dimension: dimension for Deepgrow training.  It can be 2 or 3.
+        image_key: image key in input datalist. Defaults to 'image'.
+        label_key: label key in input datalist. Defaults to 'label'.
+        base_dir: base directory in case related path is used for the keys in datalist.  Defaults to None.
+        limit: limit number of inputs for pre-processing.  Defaults to 0 (no limit).
+        relative_path: output keys values should be based on relative path.  Defaults to False.
+        transforms: explicit transforms to execute operations on input data.
+
+    Raises:
+        ValueError: When ``dimension`` is not one of [2, 3]
+        ValueError: When ``datalist`` is Empty
+
+    Returns:
+        A new datalist that contains path to the images/labels after pre-processing.
+
+    Example::
+
+        datalist = create_dataset(
+            datalist=[{'image': 'img1.nii', 'label': 'label1.nii'}],
+            base_dir=None,
+            output_dir=output_2d,
+            dimension=2,
+            image_key='image',
+            label_key='label',
+            pixdim=(1.0, 1.0),
+            limit=0,
+            relative_path=True
+        )
+
+        print(datalist[0]["image"], datalist[0]["label"])
+    """
+
+    if dimension not in [2, 3]:
+        raise ValueError("Dimension can be only 2 or 3 as Deepgrow supports only 2D/3D Training")
+
+    if not len(datalist):
+        raise ValueError("Input datalist is empty")
+
+    transforms = _default_transforms(image_key, label_key, pixdim) if transforms is None else transforms
+    new_datalist = []
+    for idx in range(len(datalist)):
+        if limit and idx >= limit:
+            break
+
+        image = datalist[idx][image_key]
+        label = datalist[idx].get(label_key, None)
+        if base_dir:
+            image = os.path.join(base_dir, image)
+            label = os.path.join(base_dir, label) if label else None
+
+        image = os.path.abspath(image)
+        label = os.path.abspath(label) if label else None
+
+        logging.info("Image: {}; Label: {}".format(image, label if label else None))
+        data = transforms({image_key: image, label_key: label})
+        if dimension == 2:
+            data = _save_data_2d(
+                vol_idx=idx,
+                vol_image=data[image_key],
+                vol_label=data[label_key],
+                dataset_dir=output_dir,
+                relative_path=relative_path,
+            )
+        else:
+            data = _save_data_3d(
+                vol_idx=idx,
+                vol_image=data[image_key],
+                vol_label=data[label_key],
+                dataset_dir=output_dir,
+                relative_path=relative_path,
+            )
+        new_datalist.extend(data)
+    return new_datalist
+
+
+def _default_transforms(image_key, label_key, pixdim):
+    keys = [image_key] if label_key is None else [image_key, label_key]
+    mode = [GridSampleMode.BILINEAR, GridSampleMode.NEAREST] if len(keys) == 2 else [GridSampleMode.BILINEAR]
+    return Compose(
+        [
+            LoadImaged(keys=keys),
+            AsChannelFirstd(keys=keys),
+            Spacingd(keys=keys, pixdim=pixdim, mode=mode),
+            Orientationd(keys=keys, axcodes="RAS"),
+        ]
+    )
+
+
+def _save_data_2d(vol_idx, vol_image, vol_label, dataset_dir, relative_path):
+    data_list = []
+
+    if len(vol_image.shape) == 4:
+        logging.info(
+            "4D-Image, pick only first series; Image: {}; Label: {}".format(
+                vol_image.shape, vol_label.shape if vol_label else None
+            )
+        )
+        vol_image = vol_image[0]
+        vol_image = np.moveaxis(vol_image, -1, 0)
+
+    image_count = 0
+    label_count = 0
+    unique_labels_count = 0
+    for sid in range(vol_image.shape[0]):
+        image = vol_image[sid, ...]
+        label = vol_label[sid, ...] if vol_label is not None else None
+
+        if vol_label is not None and np.sum(label) == 0:
+            continue
+
+        image_file_prefix = "vol_idx_{:0>4d}_slice_{:0>3d}".format(vol_idx, sid)
+        image_file = os.path.join(dataset_dir, "images", image_file_prefix)
+        image_file += ".npy"
+
+        os.makedirs(os.path.join(dataset_dir, "images"), exist_ok=True)
+        np.save(image_file, image)
+        image_count += 1
+
+        # Test Data
+        if vol_label is None:
+            data_list.append(
+                {
+                    "image": image_file.replace(dataset_dir + os.pathsep, "") if relative_path else image_file,
+                }
+            )
+            continue
+
+        # For all Labels
+        unique_labels = np.unique(label.flatten())
+        unique_labels = unique_labels[unique_labels != 0]
+        unique_labels_count = max(unique_labels_count, len(unique_labels))
+
+        for idx in unique_labels:
+            label_file_prefix = "{}_region_{:0>2d}".format(image_file_prefix, int(idx))
+            label_file = os.path.join(dataset_dir, "labels", label_file_prefix)
+            label_file += ".npy"
+
+            os.makedirs(os.path.join(dataset_dir, "labels"), exist_ok=True)
+            curr_label = (label == idx).astype(np.float32)
+            np.save(label_file, curr_label)
+
+            label_count += 1
+            data_list.append(
+                {
+                    "image": image_file.replace(dataset_dir + os.pathsep, "") if relative_path else image_file,
+                    "label": label_file.replace(dataset_dir + os.pathsep, "") if relative_path else label_file,
+                    "region": int(idx),
+                }
+            )
+
+    if unique_labels_count >= 20:
+        logging.warning(f"Unique labels {unique_labels_count} exceeds 20. Please check if this is correct.")
+
+    logging.info(
+        "{} => Image Shape: {} => {}; Label Shape: {} => {}; Unique Labels: {}".format(
+            vol_idx,
+            vol_image.shape,
+            image_count,
+            vol_label.shape if vol_label is not None else None,
+            label_count,
+            unique_labels_count,
+        )
+    )
+    return data_list
+
+
+def _save_data_3d(vol_idx, vol_image, vol_label, dataset_dir, relative_path):
+    data_list = []
+
+    if len(vol_image.shape) == 4:
+        logging.info(
+            "4D-Image, pick only first series; Image: {}; Label: {}".format(
+                vol_image.shape, vol_label.shape if vol_label else None
+            )
+        )
+        vol_image = vol_image[0]
+        vol_image = np.moveaxis(vol_image, -1, 0)
+
+    image_count = 0
+    label_count = 0
+    unique_labels_count = 0
+
+    image_file_prefix = "vol_idx_{:0>4d}".format(vol_idx)
+    image_file = os.path.join(dataset_dir, "images", image_file_prefix)
+    image_file += ".npy"
+
+    os.makedirs(os.path.join(dataset_dir, "images"), exist_ok=True)
+    np.save(image_file, vol_image)
+    image_count += 1
+
+    # Test Data
+    if vol_label is None:
+        data_list.append(
+            {
+                "image": image_file.replace(dataset_dir + os.pathsep, "") if relative_path else image_file,
+            }
+        )
+    else:
+        # For all Labels
+        unique_labels = np.unique(vol_label.flatten())
+        unique_labels = unique_labels[unique_labels != 0]
+        unique_labels_count = max(unique_labels_count, len(unique_labels))
+
+        for idx in unique_labels:
+            label_file_prefix = "{}_region_{:0>2d}".format(image_file_prefix, int(idx))
+            label_file = os.path.join(dataset_dir, "labels", label_file_prefix)
+            label_file += ".npy"
+
+            curr_label = (vol_label == idx).astype(np.float32)
+            os.makedirs(os.path.join(dataset_dir, "labels"), exist_ok=True)
+            np.save(label_file, curr_label)
+
+            label_count += 1
+            data_list.append(
+                {
+                    "image": image_file.replace(dataset_dir + os.pathsep, "") if relative_path else image_file,
+                    "label": label_file.replace(dataset_dir + os.pathsep, "") if relative_path else label_file,
+                    "region": int(idx),
+                }
+            )
+
+    if unique_labels_count >= 20:
+        logging.warning(f"Unique labels {unique_labels_count} exceeds 20. Please check if this is correct.")
+
+    logging.info(
+        "{} => Image Shape: {} => {}; Label Shape: {} => {}; Unique Labels: {}".format(
+            vol_idx,
+            vol_image.shape,
+            image_count,
+            vol_label.shape if vol_label is not None else None,
+            label_count,
+            unique_labels_count,
+        )
+    )
+    return data_list
diff --git a/monai/apps/deepgrow/interaction.py b/monai/apps/deepgrow/interaction.py
new file mode 100644
index 0000000000..77e271a9eb
--- /dev/null
+++ b/monai/apps/deepgrow/interaction.py
@@ -0,0 +1,79 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Callable, Dict, Sequence, Union
+
+import torch
+
+from monai.engines import SupervisedEvaluator, SupervisedTrainer
+from monai.engines.utils import CommonKeys
+from monai.engines.workflow import Events
+from monai.transforms import Compose
+
+
+class Interaction:
+    """
+    Ignite handler used to introduce interactions (simulation of clicks) for Deepgrow Training/Evaluation.
+    This implementation is based on:
+
+        Sakinis et al., Interactive segmentation of medical images through
+        fully convolutional neural networks. (2019) https://arxiv.org/abs/1903.08205
+
+    Args:
+        transforms: execute additional transformation during every iteration (before train).
+            Typically, several Tensor based transforms composed by `Compose`.
+        max_interactions: maximum number of interactions per iteration
+        train: training or evaluation
+        key_probability: field name to fill probability for every interaction
+    """
+
+    def __init__(
+        self,
+        transforms: Union[Sequence[Callable], Callable],
+        max_interactions: int,
+        train: bool,
+        key_probability: str = "probability",
+    ) -> None:
+
+        if not isinstance(transforms, Compose):
+            transforms = Compose(transforms)
+
+        self.transforms = transforms
+        self.max_interactions = max_interactions
+        self.train = train
+        self.key_probability = key_probability
+
+    def attach(self, engine: Union[SupervisedTrainer, SupervisedEvaluator]) -> None:
+        if not engine.has_event_handler(self, Events.ITERATION_STARTED):
+            engine.add_event_handler(Events.ITERATION_STARTED, self)
+
+    def __call__(self, engine: Union[SupervisedTrainer, SupervisedEvaluator], batchdata: Dict[str, torch.Tensor]):
+        if batchdata is None:
+            raise ValueError("Must provide batch data for current iteration.")
+
+        for j in range(self.max_interactions):
+            inputs, _ = engine.prepare_batch(batchdata)
+            inputs = inputs.to(engine.state.device)
+
+            engine.network.eval()
+            with torch.no_grad():
+                if engine.amp:
+                    with torch.cuda.amp.autocast():
+                        predictions = engine.inferer(inputs, engine.network)
+                else:
+                    predictions = engine.inferer(inputs, engine.network)
+
+            batchdata.update({CommonKeys.PRED: predictions})
+            batchdata[self.key_probability] = torch.as_tensor(
+                ([1.0 - ((1.0 / self.max_interactions) * j)] if self.train else [1.0]) * len(inputs)
+            )
+            batchdata = self.transforms(batchdata)
+
+        return engine._iteration(engine, batchdata)
diff --git a/monai/apps/deepgrow/transforms.py b/monai/apps/deepgrow/transforms.py
new file mode 100644
index 0000000000..f178360031
--- /dev/null
+++ b/monai/apps/deepgrow/transforms.py
@@ -0,0 +1,479 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Callable, Optional, Sequence, Union
+
+import numpy as np
+import torch
+
+from monai.config import IndexSelection, KeysCollection
+from monai.networks.layers import GaussianFilter
+from monai.transforms import SpatialCrop
+from monai.transforms.compose import MapTransform, Randomizable, Transform
+from monai.transforms.utils import generate_spatial_bounding_box
+from monai.utils import min_version, optional_import
+
+measure, _ = optional_import("skimage.measure", "0.14.2", min_version)
+distance_transform_cdt, _ = optional_import("scipy.ndimage.morphology", name="distance_transform_cdt")
+
+
+# Transforms to support Training for Deepgrow models
+class FindAllValidSlicesd(Transform):
+    """
+    Find/List all valid slices in the label.
+    Label is assumed to be a 4D Volume with shape CDHW, where C=1.
+
+    Args:
+        label: key to the label source.
+        sids: key to store slices indices having valid label map.
+    """
+
+    def __init__(self, label: str = "label", sids: str = "sids"):
+        self.label = label
+        self.sids = sids
+
+    def _apply(self, label):
+        sids = []
+        for sid in range(label.shape[1]):  # Assume channel is first
+            if np.sum(label[0][sid]) != 0:
+                sids.append(sid)
+        return np.asarray(sids)
+
+    def __call__(self, data):
+        d = dict(data)
+        label = d[self.label]
+        if label.shape[0] != 1:
+            raise ValueError("Only supports single channel labels!")
+
+        if len(label.shape) != 4:  # only for 3D
+            raise ValueError("Only supports label with shape CDHW!")
+
+        sids = self._apply(label)
+        if sids is not None and len(sids):
+            d[self.sids] = sids
+        return d
+
+
+class AddInitialSeedPointd(Randomizable, Transform):
+    """
+    Add random guidance as initial seed point for a given label.
+
+    Note that the label is of size (C, D, H, W) or (C, H, W)
+    The guidance is of size (2, N, # of dims) where N is number of guidance added
+    # of dims = 4 when C, D, H, W; # of dims = 3 when (C, H, W)
+
+    Args:
+        label: label source.
+        guidance: key to store guidance.
+        sids: key that represents list of valid slice indices for the given label.
+        sid: key that represents the slice to add initial seed point.  If not present, random sid will be chosen.
+        connected_regions: maximum connected regions to use for adding initial points.
+    """
+
+    def __init__(
+        self,
+        label: str = "label",
+        guidance: str = "guidance",
+        sids: str = "sids",
+        sid: str = "sid",
+        connected_regions: int = 5,
+    ):
+        self.label = label
+        self.sids = sids
+        self.sid = sid
+        self.guidance = guidance
+        self.connected_regions = connected_regions
+
+    def randomize(self, data=None):
+        pass
+
+    def _apply(self, label, sid):
+        dimensions = 3 if len(label.shape) > 3 else 2
+        default_guidance = [-1] * (dimensions + 1)
+
+        dims = dimensions
+        if sid is not None and dimensions == 3:
+            dims = 2
+            label = label[0][sid][np.newaxis]  # Assume channel is first
+
+        label = (label > 0.5).astype(np.float32)
+        blobs_labels = measure.label(label.astype(int), background=0) if dims == 2 else label
+        assert np.max(blobs_labels) > 0, "Not a valid Label"
+
+        pos_guidance = []
+        for ridx in range(1, 2 if dims == 3 else self.connected_regions + 1):
+            if dims == 2:
+                label = (blobs_labels == ridx).astype(np.float32)
+                if np.sum(label) == 0:
+                    pos_guidance.append(default_guidance)
+                    continue
+
+            distance = distance_transform_cdt(label).flatten()
+            probability = np.exp(distance) - 1.0
+
+            idx = np.where(label.flatten() > 0)[0]
+            seed = self.R.choice(idx, size=1, p=probability[idx] / np.sum(probability[idx]))
+            dst = distance[seed]
+
+            g = np.asarray(np.unravel_index(seed, label.shape)).transpose().tolist()[0]
+            g[0] = dst[0]  # for debug
+            if dimensions == 2 or dims == 3:
+                pos_guidance.append(g)
+            else:
+                pos_guidance.append([g[0], sid, g[-2], g[-1]])
+
+        return np.asarray([pos_guidance, [default_guidance] * len(pos_guidance)])
+
+    def __call__(self, data):
+        d = dict(data)
+        sid = d.get(self.sid, None)
+        sids = d.get(self.sids, None)
+        if sids is not None:
+            if sid is None or sid not in sids:
+                sid = self.R.choice(sids, replace=False)
+        else:
+            sid = None
+        d[self.guidance] = self._apply(d[self.label], sid)
+        return d
+
+
+class AddGuidanceSignald(Transform):
+    """
+    Add Guidance signal for input image.
+
+    Based on the "guidance" points, apply gaussian to them and add them as new channel for input image.
+
+    Args:
+        image: key to the image source.
+        guidance: key to store guidance.
+        sigma: standard deviation for Gaussian kernel.
+        number_intensity_ch: channel index.
+        batched: whether input is batched or not.
+    """
+
+    def __init__(
+        self,
+        image: str = "image",
+        guidance: str = "guidance",
+        sigma: int = 2,
+        number_intensity_ch: int = 1,
+        batched: bool = False,
+    ):
+        self.image = image
+        self.guidance = guidance
+        self.sigma = sigma
+        self.number_intensity_ch = number_intensity_ch
+        self.batched = batched
+
+    def _get_signal(self, image, guidance):
+        dimensions = 3 if len(image.shape) > 3 else 2
+        guidance = guidance.tolist() if isinstance(guidance, np.ndarray) else guidance
+        if dimensions == 3:
+            signal = np.zeros((len(guidance), image.shape[-3], image.shape[-2], image.shape[-1]), dtype=np.float32)
+        else:
+            signal = np.zeros((len(guidance), image.shape[-2], image.shape[-1]), dtype=np.float32)
+
+        sshape = signal.shape
+        for i in range(len(guidance)):
+            for point in guidance[i]:
+                if np.any(np.asarray(point) < 0):
+                    continue
+
+                if dimensions == 3:
+                    p1 = max(0, min(int(point[-3]), sshape[-3] - 1))
+                    p2 = max(0, min(int(point[-2]), sshape[-2] - 1))
+                    p3 = max(0, min(int(point[-1]), sshape[-1] - 1))
+                    signal[i, p1, p2, p3] = 1.0
+                else:
+                    p1 = max(0, min(int(point[-2]), sshape[-2] - 1))
+                    p2 = max(0, min(int(point[-1]), sshape[-1] - 1))
+                    signal[i, p1, p2] = 1.0
+
+            if np.max(signal[i]) > 0:
+                signal_tensor = torch.tensor(signal[i])
+                pt_gaussian = GaussianFilter(len(signal_tensor.shape), sigma=self.sigma)
+                signal_tensor = pt_gaussian(signal_tensor.unsqueeze(0).unsqueeze(0))
+                signal_tensor = signal_tensor.squeeze(0).squeeze(0)
+                signal[i] = signal_tensor.detach().cpu().numpy()
+                signal[i] = (signal[i] - np.min(signal[i])) / (np.max(signal[i]) - np.min(signal[i]))
+        return signal
+
+    def _apply(self, image, guidance):
+        if not self.batched:
+            signal = self._get_signal(image, guidance)
+            return np.concatenate([image, signal], axis=0)
+
+        images = []
+        for i, g in zip(image, guidance):
+            i = i[0 : 0 + self.number_intensity_ch, ...]
+            signal = self._get_signal(i, g)
+            images.append(np.concatenate([i, signal], axis=0))
+        return images
+
+    def __call__(self, data):
+        d = dict(data)
+        image = d[self.image]
+        guidance = d[self.guidance]
+
+        d[self.image] = self._apply(image, guidance)
+        return d
+
+
+class FindDiscrepancyRegionsd(Transform):
+    """
+    Find discrepancy between prediction and actual during click interactions during training.
+
+    If batched is true:
+        label is in shape (B, C, D, H, W) or (B, C, H, W)
+        pred has same shape as label
+        discrepancy will have shape (B, 2, C, D, H, W) or (B, 2, C, H, W)
+
+    Args:
+        label: key to label source.
+        pred: key to prediction source.
+        discrepancy: key to store discrepancies found between label and prediction.
+        batched: whether input is batched or not.
+    """
+
+    def __init__(
+        self, label: str = "label", pred: str = "pred", discrepancy: str = "discrepancy", batched: bool = True
+    ):
+        self.label = label
+        self.pred = pred
+        self.discrepancy = discrepancy
+        self.batched = batched
+
+    @staticmethod
+    def disparity(label, pred):
+        label = (label > 0.5).astype(np.float32)
+        pred = (pred > 0.5).astype(np.float32)
+        disparity = label - pred
+
+        pos_disparity = (disparity > 0).astype(np.float32)
+        neg_disparity = (disparity < 0).astype(np.float32)
+        return [pos_disparity, neg_disparity]
+
+    def _apply(self, label, pred):
+        if not self.batched:
+            return self.disparity(label, pred)
+
+        disparity = []
+        for la, pr in zip(label, pred):
+            disparity.append(self.disparity(la, pr))
+        return disparity
+
+    def __call__(self, data):
+        d = dict(data)
+        label = d[self.label]
+        pred = d[self.pred]
+
+        d[self.discrepancy] = self._apply(label, pred)
+        return d
+
+
+class AddRandomGuidanced(Randomizable, Transform):
+    """
+    Add random guidance based on discrepancies that were found between label and prediction.
+
+    If batched is True:
+
+        Guidance is of shape (B, 2, N, # of dim) where B is batch size, 2 means positive and negative,
+        N means how many guidance points, # of dim is the total number of dimensions of the image
+        (for example if the image is CDHW, then # of dim would be 4).
+
+        Discrepancy is of shape (B, 2, C, D, H, W) or (B, 2, C, H, W)
+
+        Probability is of shape (B,)
+
+    Args:
+        guidance: key to guidance source.
+        discrepancy: key that represents discrepancies found between label and prediction.
+        probability: key that represents click/interaction probability.
+        batched: whether input is batched or not.
+    """
+
+    def __init__(
+        self,
+        guidance: str = "guidance",
+        discrepancy: str = "discrepancy",
+        probability: str = "probability",
+        batched: bool = True,
+    ):
+        self.guidance = guidance
+        self.discrepancy = discrepancy
+        self.probability = probability
+        self.batched = batched
+        self._will_interact = None
+
+    def randomize(self, data=None):
+        probability = data[self.probability]
+        if not self.batched:
+            self._will_interact = self.R.choice([True, False], p=[probability, 1.0 - probability])
+        else:
+            self._will_interact = []
+            for p in probability:
+                self._will_interact.append(self.R.choice([True, False], p=[p, 1.0 - p]))
+
+    def find_guidance(self, discrepancy):
+        distance = distance_transform_cdt(discrepancy).flatten()
+        probability = np.exp(distance) - 1.0
+        idx = np.where(discrepancy.flatten() > 0)[0]
+
+        if np.sum(discrepancy > 0) > 0:
+            seed = self.R.choice(idx, size=1, p=probability[idx] / np.sum(probability[idx]))
+            dst = distance[seed]
+
+            g = np.asarray(np.unravel_index(seed, discrepancy.shape)).transpose().tolist()[0]
+            g[0] = dst[0]
+            return g
+        return None
+
+    def add_guidance(self, discrepancy, will_interact):
+        if not will_interact:
+            return None, None
+
+        pos_discr = discrepancy[0]
+        neg_discr = discrepancy[1]
+
+        can_be_positive = np.sum(pos_discr) > 0
+        can_be_negative = np.sum(neg_discr) > 0
+        correct_pos = np.sum(pos_discr) >= np.sum(neg_discr)
+
+        if correct_pos and can_be_positive:
+            return self.find_guidance(pos_discr), None
+
+        if not correct_pos and can_be_negative:
+            return None, self.find_guidance(neg_discr)
+        return None, None
+
+    def _apply(self, guidance, discrepancy):
+        guidance = guidance.tolist() if isinstance(guidance, np.ndarray) else guidance
+        if not self.batched:
+            pos, neg = self.add_guidance(discrepancy, self._will_interact)
+            if pos:
+                guidance[0].append(pos)
+                guidance[1].append([-1] * len(pos))
+            if neg:
+                guidance[0].append([-1] * len(neg))
+                guidance[1].append(neg)
+        else:
+            for g, d, w in zip(guidance, discrepancy, self._will_interact):
+                pos, neg = self.add_guidance(d, w)
+                if pos:
+                    g[0].append(pos)
+                    g[1].append([-1] * len(pos))
+                if neg:
+                    g[0].append([-1] * len(neg))
+                    g[1].append(neg)
+        return np.asarray(guidance)
+
+    def __call__(self, data):
+        d = dict(data)
+        guidance = d[self.guidance]
+        discrepancy = d[self.discrepancy]
+
+        self.randomize(data)
+        d[self.guidance] = self._apply(guidance, discrepancy)
+        return d
+
+
+class SpatialCropForegroundd(MapTransform):
+    """
+    Crop only the foreground object of the expected images.
+
+    Difference VS CropForegroundd:
+
+      1. If the bounding box is smaller than spatial size in all dimensions then this transform will crop the
+         object using box's center and spatial_size.
+
+      2. This transform will set "start_coord_key", "end_coord_key", "original_shape_key" and "cropped_shape_key"
+         in data[{key}_{meta_key_postfix}]
+
+    The typical usage is to help training and evaluation if the valid part is small in the whole medical image.
+    The valid part can be determined by any field in the data with `source_key`, for example:
+    - Select values > 0 in image field as the foreground and crop on all fields specified by `keys`.
+    - Select label = 3 in label field as the foreground to crop on all fields specified by `keys`.
+    - Select label > 0 in the third channel of a One-Hot label field as the foreground to crop all `keys` fields.
+    Users can define arbitrary function to select expected foreground from the whole source image or specified
+    channels. And it can also add margin to every dim of the bounding box of foreground object.
+
+    Args:
+        keys: keys of the corresponding items to be transformed.
+            See also: :py:class:`monai.transforms.MapTransform`
+        source_key: data source to generate the bounding box of foreground, can be image or label, etc.
+        spatial_size: minimal spatial size of the image patch e.g. [128, 128, 128] to fit in.
+        select_fn: function to select expected foreground, default is to select values > 0.
+        channel_indices: if defined, select foreground only on the specified channels
+            of image. if None, select foreground on the whole image.
+        margin: add margin value to spatial dims of the bounding box, if only 1 value provided, use it for all dims.
+        meta_key_postfix: use `{key}_{meta_key_postfix}` to to fetch/store the meta data according to the key data,
+            default is `meta_dict`, the meta data is a dictionary object.
+            For example, to handle key `image`,  read/write affine matrices from the
+            metadata `image_meta_dict` dictionary's `affine` field.
+        start_coord_key: key to record the start coordinate of spatial bounding box for foreground.
+        end_coord_key: key to record the end coordinate of spatial bounding box for foreground.
+        original_shape_key: key to record original shape for foreground.
+        cropped_shape_key: key to record cropped shape for foreground.
+    """
+
+    def __init__(
+        self,
+        keys: KeysCollection,
+        source_key: str,
+        spatial_size: Union[Sequence[int], np.ndarray],
+        select_fn: Callable = lambda x: x > 0,
+        channel_indices: Optional[IndexSelection] = None,
+        margin: int = 0,
+        meta_key_postfix="meta_dict",
+        start_coord_key: str = "foreground_start_coord",
+        end_coord_key: str = "foreground_end_coord",
+        original_shape_key: str = "foreground_original_shape",
+        cropped_shape_key: str = "foreground_cropped_shape",
+    ) -> None:
+        super().__init__(keys)
+
+        self.source_key = source_key
+        self.spatial_size = list(spatial_size)
+        self.select_fn = select_fn
+        self.channel_indices = channel_indices
+        self.margin = margin
+        self.meta_key_postfix = meta_key_postfix
+        self.start_coord_key = start_coord_key
+        self.end_coord_key = end_coord_key
+        self.original_shape_key = original_shape_key
+        self.cropped_shape_key = cropped_shape_key
+
+    def __call__(self, data):
+        d = dict(data)
+        box_start, box_end = generate_spatial_bounding_box(
+            d[self.source_key], self.select_fn, self.channel_indices, self.margin
+        )
+
+        center = np.mean([box_start, box_end], axis=0).astype(int).tolist()
+        current_size = np.subtract(box_end, box_start).astype(int).tolist()
+
+        if np.all(np.less(current_size, self.spatial_size)):
+            cropper = SpatialCrop(roi_center=center, roi_size=self.spatial_size)
+            box_start = cropper.roi_start
+            box_end = cropper.roi_end
+        else:
+            cropper = SpatialCrop(roi_start=box_start, roi_end=box_end)
+
+        for key in self.keys:
+            meta_key = f"{key}_{self.meta_key_postfix}"
+            d[meta_key][self.start_coord_key] = box_start
+            d[meta_key][self.end_coord_key] = box_end
+            d[meta_key][self.original_shape_key] = d[key].shape
+
+            image = cropper(d[key])
+            d[meta_key][self.cropped_shape_key] = image.shape
+            d[key] = image
+        return d
diff --git a/monai/config/__init__.py b/monai/config/__init__.py
index 251be002f2..f1c7707d1f 100644
--- a/monai/config/__init__.py
+++ b/monai/config/__init__.py
@@ -18,4 +18,4 @@
     print_gpu_info,
     print_system_info,
 )
-from .type_definitions import IndexSelection, KeysCollection
+from .type_definitions import DtypeLike, IndexSelection, KeysCollection, NdarrayTensor
diff --git a/monai/config/deviceconfig.py b/monai/config/deviceconfig.py
index 9e448a9ac3..be77a1d975 100644
--- a/monai/config/deviceconfig.py
+++ b/monai/config/deviceconfig.py
@@ -162,7 +162,7 @@ def get_system_info() -> OrderedDict:
             _dict_append(
                 output,
                 "Avg. sensor temp. (Celsius)",
-                lambda: round(
+                lambda: np.round(
                     np.mean([item.current for sublist in psutil.sensors_temperatures().values() for item in sublist], 1)
                 ),
             )
diff --git a/monai/config/type_definitions.py b/monai/config/type_definitions.py
index ea0c72576c..daa9b10052 100644
--- a/monai/config/type_definitions.py
+++ b/monai/config/type_definitions.py
@@ -9,9 +9,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Collection, Hashable, Iterable, Union
+from typing import Collection, Hashable, Iterable, TypeVar, Union
 
-__all__ = ["KeysCollection", "IndexSelection"]
+import numpy as np
+import torch
+
+__all__ = ["KeysCollection", "IndexSelection", "DtypeLike", "NdarrayTensor"]
 
 """Commonly used concepts
 This module provides naming and type specifications for commonly used concepts
@@ -51,3 +54,16 @@
 The indices must be integers, and if a container of indices is specified, the
 container must be iterable.
 """
+
+DtypeLike = Union[
+    np.dtype,
+    type,
+    None,
+]
+"""Type of datatypes
+adapted from https://github.com/numpy/numpy/blob/master/numpy/typing/_dtype_like.py
+"""
+
+# Generic type which can represent either a numpy.ndarray or a torch.Tensor
+# Unlike Union can create a dependence between parameter(s) / return(s)
+NdarrayTensor = TypeVar("NdarrayTensor", np.ndarray, torch.Tensor)
diff --git a/monai/data/csv_saver.py b/monai/data/csv_saver.py
index 5f5e415055..830c6a4f0d 100644
--- a/monai/data/csv_saver.py
+++ b/monai/data/csv_saver.py
@@ -17,6 +17,8 @@
 import numpy as np
 import torch
 
+from monai.utils import ImageMetaKey as Key
+
 
 class CSVSaver:
     """
@@ -73,9 +75,9 @@ def save(self, data: Union[torch.Tensor, np.ndarray], meta_data: Optional[Dict]
             meta_data: the meta data information corresponding to the data.
 
         """
-        save_key = meta_data["filename_or_obj"] if meta_data else str(self._data_index)
+        save_key = meta_data[Key.FILENAME_OR_OBJ] if meta_data else str(self._data_index)
         self._data_index += 1
-        if torch.is_tensor(data):
+        if isinstance(data, torch.Tensor):
             data = data.detach().cpu().numpy()
         if not isinstance(data, np.ndarray):
             raise AssertionError
diff --git a/monai/data/dataset.py b/monai/data/dataset.py
index e67c7a2954..b93f03151f 100644
--- a/monai/data/dataset.py
+++ b/monai/data/dataset.py
@@ -51,14 +51,16 @@ class Dataset(_TorchDataset):
          },                           },                           }]
     """
 
-    def __init__(self, data: Sequence, transform: Optional[Callable] = None) -> None:
+    def __init__(self, data: Sequence, transform: Optional[Callable] = None, progress: bool = True) -> None:
         """
         Args:
             data: input data to load and transform to generate dataset for model.
             transform: a callable data transform on input data.
+            progress: whether to display a progress bar.
         """
         self.data = data
         self.transform = transform
+        self.progress = progress
 
     def __len__(self) -> int:
         return len(self.data)
@@ -115,6 +117,7 @@ def __init__(
         transform: Union[Sequence[Callable], Callable],
         cache_dir: Optional[Union[Path, str]] = None,
         hash_func: Callable[..., bytes] = pickle_hashing,
+        progress: bool = True,
     ) -> None:
         """
         Args:
@@ -129,10 +132,11 @@ def __init__(
                 If the cache_dir doesn't exist, will automatically create it.
             hash_func: a callable to compute hash from data items to be cached.
                 defaults to `monai.data.utils.pickle_hashing`.
+            progress: whether to display a progress bar.
         """
         if not isinstance(transform, Compose):
             transform = Compose(transform)
-        super().__init__(data=data, transform=transform)
+        super().__init__(data=data, transform=transform, progress=progress)
         self.cache_dir = Path(cache_dir) if cache_dir is not None else None
         self.hash_func = hash_func
         if self.cache_dir is not None:
@@ -345,7 +349,7 @@ def __init__(
             lmdb_kwargs: additional keyword arguments to the lmdb environment.
                 for more details please visit: https://lmdb.readthedocs.io/en/release/#environment-class
         """
-        super().__init__(data=data, transform=transform, cache_dir=cache_dir, hash_func=hash_func)
+        super().__init__(data=data, transform=transform, cache_dir=cache_dir, hash_func=hash_func, progress=progress)
         if not self.cache_dir:
             raise ValueError("cache_dir must be specified.")
         self.db_file = self.cache_dir / f"{db_name}.lmdb"
@@ -354,14 +358,13 @@ def __init__(
         if not self.lmdb_kwargs.get("map_size", 0):
             self.lmdb_kwargs["map_size"] = 1024 ** 4  # default map_size
         self._read_env = None
-        self.progress = progress
         print(f"Accessing lmdb file: {self.db_file.absolute()}.")
 
     def _fill_cache_start_reader(self):
         # create cache
         self.lmdb_kwargs["readonly"] = False
         env = lmdb.open(path=f"{self.db_file}", subdir=False, **self.lmdb_kwargs)
-        if not has_tqdm:
+        if self.progress and not has_tqdm:
             warnings.warn("LMDBDataset: tqdm is not installed. not displaying the caching progress.")
         for item in tqdm(self.data) if has_tqdm and self.progress else self.data:
             key = self.hash_func(item)
@@ -470,6 +473,7 @@ def __init__(
         cache_num: int = sys.maxsize,
         cache_rate: float = 1.0,
         num_workers: Optional[int] = None,
+        progress: bool = True,
     ) -> None:
         """
         Args:
@@ -481,10 +485,11 @@ def __init__(
                 will take the minimum of (cache_num, data_length x cache_rate, data_length).
             num_workers: the number of worker processes to use.
                 If num_workers is None then the number returned by os.cpu_count() is used.
+            progress: whether to display a progress bar.
         """
         if not isinstance(transform, Compose):
             transform = Compose(transform)
-        super().__init__(data=data, transform=transform)
+        super().__init__(data=data, transform=transform, progress=progress)
         self.cache_num = min(int(cache_num), int(len(data) * cache_rate), len(data))
         self.num_workers = num_workers
         if self.num_workers is not None:
@@ -494,10 +499,10 @@ def __init__(
     def _fill_cache(self) -> List:
         if self.cache_num <= 0:
             return []
-        if not has_tqdm:
+        if self.progress and not has_tqdm:
             warnings.warn("tqdm is not installed, will not show the caching progress bar.")
         with ThreadPool(self.num_workers) as p:
-            if has_tqdm:
+            if self.progress and has_tqdm:
                 return list(
                     tqdm(
                         p.imap(self._load_cache_item, range(self.cache_num)),
@@ -571,6 +576,10 @@ class SmartCacheDataset(CacheDataset):
         3. Call `update_cache()` before every epoch to replace training items.
         4. Call `shutdown()` when training ends.
 
+    Note:
+        This replacement will not work if setting the `multiprocessing_context` of DataLoader to `spawn`
+        or on windows(the default multiprocessing method is `spawn`) and setting `num_workers` greater than 0.
+
     """
 
     def __init__(
@@ -601,7 +610,7 @@ def __init__(
         if self._cache is None:
             self._cache = self._fill_cache()
         if self.cache_num >= len(data):
-            raise ValueError("cache_num must be smaller than dataset length to support replacement.")
+            warnings.warn("cache_num is greater or equal than dataset length, fall back to regular CacheDataset.")
         if replace_rate <= 0:
             raise ValueError("replace_rate must be greater than 0, otherwise, please use CacheDataset.")
         self.num_replace_workers: int = num_replace_workers
@@ -637,7 +646,7 @@ def is_started(self):
         """
         if self._replace_mgr is None:
             return False
-        return self._replace_mgr.isAlive()
+        return self._replace_mgr.is_alive()
 
     def start(self):
         """
@@ -688,7 +697,7 @@ def update_cache(self):
         If the cache has been shutdown before, need to restart the `_replace_mgr` thread.
 
         """
-        if not self._replace_mgr.isAlive():
+        if not self._replace_mgr.is_alive():
             self._restart()
 
         # make sure update is done
diff --git a/monai/data/image_dataset.py b/monai/data/image_dataset.py
index 7dd55431af..1568e082ee 100644
--- a/monai/data/image_dataset.py
+++ b/monai/data/image_dataset.py
@@ -14,6 +14,7 @@
 import numpy as np
 from torch.utils.data import Dataset
 
+from monai.config import DtypeLike
 from monai.data.image_reader import ImageReader
 from monai.transforms import LoadImage, Randomizable, apply_transform
 from monai.utils import MAX_SEED, get_seed
@@ -36,7 +37,7 @@ def __init__(
         transform: Optional[Callable] = None,
         seg_transform: Optional[Callable] = None,
         image_only: bool = True,
-        dtype: Optional[np.dtype] = np.float32,
+        dtype: DtypeLike = np.float32,
         reader: Optional[Union[ImageReader, str]] = None,
         *args,
         **kwargs,
diff --git a/monai/data/image_reader.py b/monai/data/image_reader.py
index 0fd784af05..e458833979 100644
--- a/monai/data/image_reader.py
+++ b/monai/data/image_reader.py
@@ -10,13 +10,14 @@
 # limitations under the License.
 
 import os
+import warnings
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Sequence, Tuple, Union
 
 import numpy as np
 from torch.utils.data._utils.collate import np_str_obj_array_pattern
 
-from monai.config import KeysCollection
+from monai.config import DtypeLike, KeysCollection
 from monai.data.utils import correct_nifti_header_if_necessary
 from monai.utils import ensure_tuple, optional_import
 
@@ -219,6 +220,16 @@ def _get_meta_dict(self, img) -> Dict:
             # ignore deprecated, legacy members that cause issues
             if key.startswith("ITK_original_"):
                 continue
+            if (
+                key == "NRRD_measurement frame"
+                and int(itk.Version.GetITKMajorVersion()) == 5
+                and int(itk.Version.GetITKMinorVersion()) < 2
+            ):
+                warnings.warn(
+                    "Ignoring 'measurement frame' field. "
+                    "Correct reading of NRRD05 files requires ITK >= 5.2: `pip install --upgrade --pre itk`"
+                )
+                continue
             meta_dict[key] = img_meta_dict[key]
         meta_dict["origin"] = np.asarray(img.GetOrigin())
         meta_dict["spacing"] = np.asarray(img.GetSpacing())
@@ -244,7 +255,7 @@ def _get_affine(self, img) -> np.ndarray:
         affine = np.eye(direction.shape[0] + 1)
         affine[(slice(-1), slice(-1))] = direction @ np.diag(spacing)
         affine[(slice(-1), -1)] = origin
-        return affine
+        return np.asarray(affine)
 
     def _get_spatial_shape(self, img) -> np.ndarray:
         """
@@ -258,7 +269,7 @@ def _get_spatial_shape(self, img) -> np.ndarray:
         shape.reverse()
         return np.asarray(shape)
 
-    def _get_array_data(self, img) -> np.ndarray:
+    def _get_array_data(self, img):
         """
         Get the raw array data of the image, converted to Numpy array.
 
@@ -295,7 +306,7 @@ class NibabelReader(ImageReader):
 
     """
 
-    def __init__(self, as_closest_canonical: bool = False, dtype: Optional[np.dtype] = np.float32, **kwargs):
+    def __init__(self, as_closest_canonical: bool = False, dtype: DtypeLike = np.float32, **kwargs):
         super().__init__()
         self.as_closest_canonical = as_closest_canonical
         self.dtype = dtype
@@ -385,7 +396,7 @@ def _get_affine(self, img) -> np.ndarray:
             img: a Nibabel image object loaded from a image file.
 
         """
-        return img.affine.copy()
+        return np.array(img.affine, copy=True)
 
     def _get_spatial_shape(self, img) -> np.ndarray:
         """
diff --git a/monai/data/nifti_saver.py b/monai/data/nifti_saver.py
index f4781f82fd..01e701b1a6 100644
--- a/monai/data/nifti_saver.py
+++ b/monai/data/nifti_saver.py
@@ -14,9 +14,11 @@
 import numpy as np
 import torch
 
+from monai.config import DtypeLike
 from monai.data.nifti_writer import write_nifti
 from monai.data.utils import create_file_basename
 from monai.utils import GridSampleMode, GridSamplePadMode
+from monai.utils import ImageMetaKey as Key
 
 
 class NiftiSaver:
@@ -36,8 +38,8 @@ def __init__(
         mode: Union[GridSampleMode, str] = GridSampleMode.BILINEAR,
         padding_mode: Union[GridSamplePadMode, str] = GridSamplePadMode.BORDER,
         align_corners: bool = False,
-        dtype: Optional[np.dtype] = np.float64,
-        output_dtype: Optional[np.dtype] = np.float32,
+        dtype: DtypeLike = np.float64,
+        output_dtype: DtypeLike = np.float32,
     ) -> None:
         """
         Args:
@@ -56,8 +58,7 @@ def __init__(
             align_corners: Geometrically, we consider the pixels of the input as squares rather than points.
                 See also: https://pytorch.org/docs/stable/nn.functional.html#grid-sample
             dtype: data type for resampling computation. Defaults to ``np.float64`` for best precision.
-                If None, use the data type of input data. To be compatible with other modules,
-                the output data type is always ``np.float32``.
+                If None, use the data type of input data.
             output_dtype: data type for saving data. Defaults to ``np.float32``.
         """
         self.output_dir = output_dir
@@ -94,13 +95,13 @@ def save(self, data: Union[torch.Tensor, np.ndarray], meta_data: Optional[Dict]
         See Also
             :py:meth:`monai.data.nifti_writer.write_nifti`
         """
-        filename = meta_data["filename_or_obj"] if meta_data else str(self._data_index)
+        filename = meta_data[Key.FILENAME_OR_OBJ] if meta_data else str(self._data_index)
         self._data_index += 1
         original_affine = meta_data.get("original_affine", None) if meta_data else None
         affine = meta_data.get("affine", None) if meta_data else None
         spatial_shape = meta_data.get("spatial_shape", None) if meta_data else None
 
-        if torch.is_tensor(data):
+        if isinstance(data, torch.Tensor):
             data = data.detach().cpu().numpy()
 
         filename = create_file_basename(self.output_postfix, filename, self.output_dir)
@@ -109,7 +110,7 @@ def save(self, data: Union[torch.Tensor, np.ndarray], meta_data: Optional[Dict]
         while len(data.shape) < 4:
             data = np.expand_dims(data, -1)
         # change data to "channel last" format and write to nifti format file
-        data = np.moveaxis(data, 0, -1)
+        data = np.moveaxis(np.asarray(data), 0, -1)
         write_nifti(
             data,
             file_name=filename,
diff --git a/monai/data/nifti_writer.py b/monai/data/nifti_writer.py
index 6837ebeb90..f530482b14 100644
--- a/monai/data/nifti_writer.py
+++ b/monai/data/nifti_writer.py
@@ -14,6 +14,7 @@
 import numpy as np
 import torch
 
+from monai.config import DtypeLike
 from monai.data.utils import compute_shape_offset, to_affine_nd
 from monai.networks.layers import AffineTransform
 from monai.utils import GridSampleMode, GridSamplePadMode, optional_import
@@ -27,12 +28,12 @@ def write_nifti(
     affine: Optional[np.ndarray] = None,
     target_affine: Optional[np.ndarray] = None,
     resample: bool = True,
-    output_spatial_shape: Optional[Sequence[int]] = None,
+    output_spatial_shape: Union[Sequence[int], np.ndarray, None] = None,
     mode: Union[GridSampleMode, str] = GridSampleMode.BILINEAR,
     padding_mode: Union[GridSamplePadMode, str] = GridSamplePadMode.BORDER,
     align_corners: bool = False,
-    dtype: Optional[np.dtype] = np.float64,
-    output_dtype: Optional[np.dtype] = np.float32,
+    dtype: DtypeLike = np.float64,
+    output_dtype: DtypeLike = np.float32,
 ) -> None:
     """
     Write numpy data into NIfTI files to disk.  This function converts data
@@ -85,8 +86,7 @@ def write_nifti(
         align_corners: Geometrically, we consider the pixels of the input as squares rather than points.
             See also: https://pytorch.org/docs/stable/nn.functional.html#grid-sample
         dtype: data type for resampling computation. Defaults to ``np.float64`` for best precision.
-            If None, use the data type of input data. To be compatible with other modules,
-            the output data type is always ``np.float32``.
+            If None, use the data type of input data.
         output_dtype: data type for saving data. Defaults to ``np.float32``.
     """
     if not isinstance(data, np.ndarray):
@@ -126,7 +126,7 @@ def write_nifti(
     transform = np.linalg.inv(_affine) @ target_affine
     if output_spatial_shape is None:
         output_spatial_shape, _ = compute_shape_offset(data.shape, _affine, target_affine)
-    output_spatial_shape_ = list(output_spatial_shape)
+    output_spatial_shape_ = list(output_spatial_shape) if output_spatial_shape is not None else []
     if data.ndim > 3:  # multi channel, resampling each channel
         while len(output_spatial_shape_) < 3:
             output_spatial_shape_ = output_spatial_shape_ + [1]
diff --git a/monai/data/png_saver.py b/monai/data/png_saver.py
index 450e327d6b..4c4c847824 100644
--- a/monai/data/png_saver.py
+++ b/monai/data/png_saver.py
@@ -16,6 +16,7 @@
 
 from monai.data.png_writer import write_png
 from monai.data.utils import create_file_basename
+from monai.utils import ImageMetaKey as Key
 from monai.utils import InterpolateMode
 
 
@@ -82,11 +83,11 @@ def save(self, data: Union[torch.Tensor, np.ndarray], meta_data: Optional[Dict]
             :py:meth:`monai.data.png_writer.write_png`
 
         """
-        filename = meta_data["filename_or_obj"] if meta_data else str(self._data_index)
+        filename = meta_data[Key.FILENAME_OR_OBJ] if meta_data else str(self._data_index)
         self._data_index += 1
         spatial_shape = meta_data.get("spatial_shape", None) if meta_data and self.resample else None
 
-        if torch.is_tensor(data):
+        if isinstance(data, torch.Tensor):
             data = data.detach().cpu().numpy()
 
         filename = create_file_basename(self.output_postfix, filename, self.output_dir)
@@ -95,12 +96,12 @@ def save(self, data: Union[torch.Tensor, np.ndarray], meta_data: Optional[Dict]
         if data.shape[0] == 1:
             data = data.squeeze(0)
         elif 2 < data.shape[0] < 5:
-            data = np.moveaxis(data, 0, -1)
+            data = np.moveaxis(np.asarray(data), 0, -1)
         else:
             raise ValueError(f"Unsupported number of channels: {data.shape[0]}, available options are [1, 3, 4]")
 
         write_png(
-            data,
+            np.asarray(data),
             file_name=filename,
             output_spatial_shape=spatial_shape,
             mode=self.mode,
diff --git a/monai/data/png_writer.py b/monai/data/png_writer.py
index d7baa6ea79..9ce01ed97f 100644
--- a/monai/data/png_writer.py
+++ b/monai/data/png_writer.py
@@ -13,7 +13,7 @@
 
 import numpy as np
 
-from monai.transforms import Resize
+from monai.transforms.spatial.array import Resize
 from monai.utils import InterpolateMode, ensure_tuple_rep, optional_import
 
 Image, _ = optional_import("PIL", name="Image")
@@ -65,10 +65,10 @@ def write_png(
             data = np.expand_dims(data, 0)  # make a channel
             data = xform(data)[0]  # first channel
         if mode != InterpolateMode.NEAREST:
-            data = np.clip(data, _min, _max)
+            data = np.clip(data, _min, _max)  # type: ignore
 
     if scale is not None:
-        data = np.clip(data, 0.0, 1.0)  # png writer only can scale data in range [0, 1]
+        data = np.clip(data, 0.0, 1.0)  # type: ignore # png writer only can scale data in range [0, 1]
         if scale == np.iinfo(np.uint8).max:
             data = (scale * data).astype(np.uint8)
         elif scale == np.iinfo(np.uint16).max:
@@ -76,6 +76,10 @@ def write_png(
         else:
             raise ValueError(f"Unsupported scale: {scale}, available options are [255, 65535]")
 
+    # PNG data must be int number
+    if data.dtype not in (np.uint8, np.uint16):  # type: ignore
+        data = data.astype(np.uint8)
+
     img = Image.fromarray(data)
     img.save(file_name, "PNG")
     return
diff --git a/monai/data/utils.py b/monai/data/utils.py
index ca8f3b1017..acc6d2e97a 100644
--- a/monai/data/utils.py
+++ b/monai/data/utils.py
@@ -329,7 +329,7 @@ def rectify_header_sform_qform(img_nii):
     return img_nii
 
 
-def zoom_affine(affine: np.ndarray, scale: Sequence[float], diagonal: bool = True) -> np.ndarray:
+def zoom_affine(affine: np.ndarray, scale: Sequence[float], diagonal: bool = True):
     """
     To make column norm of `affine` the same as `scale`.  If diagonal is False,
     returns an affine that combines orthogonal rotation and the new scale.
@@ -379,7 +379,7 @@ def zoom_affine(affine: np.ndarray, scale: Sequence[float], diagonal: bool = Tru
 
 
 def compute_shape_offset(
-    spatial_shape: np.ndarray, in_affine: np.ndarray, out_affine: np.ndarray
+    spatial_shape: Union[np.ndarray, Sequence[int]], in_affine: np.ndarray, out_affine: np.ndarray
 ) -> Tuple[np.ndarray, np.ndarray]:
     """
     Given input and output affine, compute appropriate shapes
diff --git a/monai/engines/utils.py b/monai/engines/utils.py
index f603338097..8f5899f2a5 100644
--- a/monai/engines/utils.py
+++ b/monai/engines/utils.py
@@ -32,7 +32,7 @@
 
 class IterationEvents(EventEnum):
     """
-    Addtional Events engine can register and trigger in the iteration process.
+    Additional Events engine can register and trigger in the iteration process.
     Refer to the example in ignite: https://github.com/pytorch/ignite/blob/master/ignite/engine/events.py#L146
     These Events can be triggered during training iteration:
     `FORWARD_COMPLETED` is the Event when `network(image, label)` completed.
diff --git a/monai/handlers/__init__.py b/monai/handlers/__init__.py
index 6b190518fb..8f73f7f2fd 100644
--- a/monai/handlers/__init__.py
+++ b/monai/handlers/__init__.py
@@ -24,6 +24,12 @@
 from .smartcache_handler import SmartCacheHandler
 from .stats_handler import StatsHandler
 from .surface_distance import SurfaceDistance
-from .tensorboard_handlers import TensorBoardImageHandler, TensorBoardStatsHandler
-from .utils import evenly_divisible_all_gather, stopping_fn_from_loss, stopping_fn_from_metric, write_metrics_reports
+from .tensorboard_handlers import TensorBoardHandler, TensorBoardImageHandler, TensorBoardStatsHandler
+from .utils import (
+    evenly_divisible_all_gather,
+    stopping_fn_from_loss,
+    stopping_fn_from_metric,
+    string_list_all_gather,
+    write_metrics_reports,
+)
 from .validation_handler import ValidationHandler
diff --git a/monai/handlers/checkpoint_saver.py b/monai/handlers/checkpoint_saver.py
index 8052e21cb6..1808e6b251 100644
--- a/monai/handlers/checkpoint_saver.py
+++ b/monai/handlers/checkpoint_saver.py
@@ -10,6 +10,7 @@
 # limitations under the License.
 
 import logging
+import warnings
 from typing import TYPE_CHECKING, Dict, Optional
 
 from monai.utils import exact_version, optional_import
@@ -55,6 +56,10 @@ class CheckpointSaver:
             metric in descending order.
         key_metric_filename: set a fixed filename to set the best metric model, if not None,
             `key_metric_n_saved` should be 1 and only keep the best metric model.
+        key_metric_save_state: whether to save the tracking list of key metric in the checkpoint file.
+            if `True`, then will save an object in the checkpoint file with key `checkpointer` to be consistent
+            with ignite: https://github.com/pytorch/ignite/blob/master/ignite/handlers/checkpoint.py#L99.
+            typically, it's used to resume training and compare current metric with previous N values.
         epoch_level: save checkpoint during training for every N epochs or every N iterations.
             `True` is epoch level, `False` is iteration level.
         save_interval: save checkpoint every N epochs, default is 0 to save no checkpoint.
@@ -84,6 +89,7 @@ def __init__(
         key_metric_name: Optional[str] = None,
         key_metric_n_saved: int = 1,
         key_metric_filename: Optional[str] = None,
+        key_metric_save_state: bool = False,
         epoch_level: bool = True,
         save_interval: int = 0,
         n_saved: Optional[int] = None,
@@ -156,6 +162,7 @@ def _score_func(engine: Engine):
                 score_function=_score_func,
                 score_name="key_metric",
                 n_saved=key_metric_n_saved,
+                include_self=key_metric_save_state,
             )
 
         if save_interval > 0:
@@ -172,6 +179,32 @@ def _interval_func(engine: Engine):
                 n_saved=n_saved,
             )
 
+    def load_state_dict(self, state_dict: Dict) -> None:
+        """
+        Utility to resume the internal state of key metric tracking list if configured to save
+        checkpoints based on the key metric value.
+        Note to set `key_metric_save_state=True` when saving the previous checkpoint.
+
+        Example::
+
+            CheckpointSaver(
+                ...
+                save_key_metric=True,
+                key_metric_save_state=True,  # config to also save the state of this saver
+            ).attach(engine)
+            engine.run(...)
+
+            # resumed training with a new CheckpointSaver
+            saver = CheckpointSaver(save_key_metric=True, ...)
+            # load the previous key metric tracking list into saver
+            CheckpointLoader("/test/model.pt"), {"checkpointer": saver}).attach(engine)
+
+        """
+        if self._key_metric_checkpoint is not None:
+            self._key_metric_checkpoint.load_state_dict(state_dict)
+        else:
+            warnings.warn("no key metric checkpoint saver to resume the key metric tracking list.")
+
     def attach(self, engine: Engine) -> None:
         """
         Args:
diff --git a/monai/handlers/classification_saver.py b/monai/handlers/classification_saver.py
index 6753cafcb0..33ce7c7ec8 100644
--- a/monai/handlers/classification_saver.py
+++ b/monai/handlers/classification_saver.py
@@ -13,8 +13,11 @@
 from typing import TYPE_CHECKING, Callable, Optional
 
 from monai.data import CSVSaver
+from monai.handlers.utils import evenly_divisible_all_gather, string_list_all_gather
+from monai.utils import ImageMetaKey as Key
 from monai.utils import exact_version, optional_import
 
+idist, _ = optional_import("ignite", "0.4.2", exact_version, "distributed")
 Events, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Events")
 if TYPE_CHECKING:
     from ignite.engine import Engine
@@ -25,6 +28,8 @@
 class ClassificationSaver:
     """
     Event handler triggered on completing every iteration to save the classification predictions as CSV file.
+    If running in distributed data parallel, only saves CSV file in the specified rank.
+
     """
 
     def __init__(
@@ -35,6 +40,7 @@ def __init__(
         batch_transform: Callable = lambda x: x,
         output_transform: Callable = lambda x: x,
         name: Optional[str] = None,
+        save_rank: int = 0,
     ) -> None:
         """
         Args:
@@ -49,8 +55,11 @@ def __init__(
                 The first dimension of this transform's output will be treated as the
                 batch dimension. Each item in the batch will be saved individually.
             name: identifier of logging.logger to use, defaulting to `engine.logger`.
+            save_rank: only the handler on specified rank will save to CSV file in multi-gpus validation,
+                default to 0.
 
         """
+        self._expected_rank: bool = idist.get_rank() == save_rank
         self.saver = CSVSaver(output_dir, filename, overwrite)
         self.batch_transform = batch_transform
         self.output_transform = output_transform
@@ -67,7 +76,7 @@ def attach(self, engine: Engine) -> None:
             self.logger = engine.logger
         if not engine.has_event_handler(self, Events.ITERATION_COMPLETED):
             engine.add_event_handler(Events.ITERATION_COMPLETED, self)
-        if not engine.has_event_handler(self.saver.finalize, Events.COMPLETED):
+        if self._expected_rank and not engine.has_event_handler(self.saver.finalize, Events.COMPLETED):
             engine.add_event_handler(Events.COMPLETED, lambda engine: self.saver.finalize())
 
     def __call__(self, engine: Engine) -> None:
@@ -77,6 +86,12 @@ def __call__(self, engine: Engine) -> None:
         Args:
             engine: Ignite Engine, it can be a trainer, validator or evaluator.
         """
-        meta_data = self.batch_transform(engine.state.batch)
-        engine_output = self.output_transform(engine.state.output)
-        self.saver.save_batch(engine_output, meta_data)
+        _meta_data = self.batch_transform(engine.state.batch)
+        if Key.FILENAME_OR_OBJ in _meta_data:
+            # all gather filenames across ranks, only filenames are necessary
+            _meta_data = {Key.FILENAME_OR_OBJ: string_list_all_gather(_meta_data[Key.FILENAME_OR_OBJ])}
+        # all gather predictions across ranks
+        _engine_output = evenly_divisible_all_gather(self.output_transform(engine.state.output))
+
+        if self._expected_rank:
+            self.saver.save_batch(_engine_output, _meta_data)
diff --git a/monai/handlers/iteration_metric.py b/monai/handlers/iteration_metric.py
index bfc7252b2f..641efad243 100644
--- a/monai/handlers/iteration_metric.py
+++ b/monai/handlers/iteration_metric.py
@@ -96,7 +96,7 @@ def compute(self) -> Any:
         # save score of every image into engine.state for other components
         if self.save_details:
             if self._engine is None or self._name is None:
-                raise RuntimeError("plesae call the attach() function to connect expected engine first.")
+                raise RuntimeError("please call the attach() function to connect expected engine first.")
             self._engine.state.metric_details[self._name] = _scores
 
         result: torch.Tensor = torch.zeros(1)
@@ -108,7 +108,7 @@ def compute(self) -> Any:
             # broadcast result to all processes
             result = idist.broadcast(result, src=0)
 
-        return result.item() if torch.is_tensor(result) else result
+        return result.item() if isinstance(result, torch.Tensor) else result
 
     def _reduce(self, scores) -> Any:
         return do_metric_reduction(scores, MetricReduction.MEAN)[0]
diff --git a/monai/handlers/metrics_saver.py b/monai/handlers/metrics_saver.py
index f9deea35df..87d7223c96 100644
--- a/monai/handlers/metrics_saver.py
+++ b/monai/handlers/metrics_saver.py
@@ -11,9 +11,9 @@
 
 from typing import TYPE_CHECKING, Callable, List, Optional, Sequence, Union
 
-from monai.handlers.utils import write_metrics_reports
+from monai.handlers.utils import string_list_all_gather, write_metrics_reports
+from monai.utils import ImageMetaKey as Key
 from monai.utils import ensure_tuple, exact_version, optional_import
-from monai.utils.module import get_torch_version_tuple
 
 Events, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Events")
 idist, _ = optional_import("ignite", "0.4.2", exact_version, "distributed")
@@ -34,26 +34,28 @@ class MetricsSaver:
             "*" - save all the existing metrics in `engine.state.metrics` dict into separate files.
             list of strings - specify the expected metrics to save.
             default to "*" to save all the metrics into `metrics.csv`.
-        metric_details: expected metric details to save into files, for example: mean dice
-            of every channel of every image in the validation dataset.
-            the data in `engine.state.metric_details` must contain at least 2 dims: (batch, classes, ...),
+        metric_details: expected metric details to save into files, the data comes from
+            `engine.state.metric_details`, which should be provided by different `Metrics`,
+            typically, it's some intermediate values in metric computation.
+            for example: mean dice of every channel of every image in the validation dataset.
+            it must contain at least 2 dims: (batch, classes, ...),
             if not, will unsequeeze to 2 dims.
             this arg can be: None, "*" or list of strings.
-            None - don't save any metrics into files.
-            "*" - save all the existing metrics in `engine.state.metric_details` dict into separate files.
-            list of strings - specify the expected metrics to save.
-            if not None, every metric will save a separate `{metric name}_raw.csv` file.
+            None - don't save any metric_details into files.
+            "*" - save all the existing metric_details in `engine.state.metric_details` dict into separate files.
+            list of strings - specify the metric_details of expected metrics to save.
+            if not None, every metric_details array will save a separate `{metric name}_raw.csv` file.
         batch_transform: callable function to extract the meta_dict from input batch data if saving metric details.
             used to extract filenames from input dict data.
-        summary_ops: expected computation operations to generate the summary report.
+        summary_ops: expected computation operations to generate the summary report based on specified metric_details.
             it can be: None, "*" or list of strings.
-            None - don't generate summary report for every expected metric_details
+            None - don't generate summary report for every specified metric_details
             "*" - generate summary report for every metric_details with all the supported operations.
             list of strings - generate summary report for every metric_details with specified operations, they
             should be within this list: [`mean`, `median`, `max`, `min`, `90percent`, `std`].
             default to None.
         save_rank: only the handler on specified rank will save to files in multi-gpus validation, default to 0.
-        delimiter: the delimiter charactor in CSV file, default to "\t".
+        delimiter: the delimiter character in CSV file, default to "\t".
         output_type: expected output file type, supported types: ["csv"], default to "csv".
 
     """
@@ -93,7 +95,7 @@ def _started(self, engine: Engine) -> None:
 
     def _get_filenames(self, engine: Engine) -> None:
         if self.metric_details is not None:
-            _filenames = list(ensure_tuple(self.batch_transform(engine.state.batch)["filename_or_obj"]))
+            _filenames = list(ensure_tuple(self.batch_transform(engine.state.batch)[Key.FILENAME_OR_OBJ]))
             self._filenames += _filenames
 
     def __call__(self, engine: Engine) -> None:
@@ -105,15 +107,8 @@ def __call__(self, engine: Engine) -> None:
         if self.save_rank >= ws:
             raise ValueError("target rank is greater than the distributed group size.")
 
-        _images = self._filenames
-        if ws > 1:
-            _filenames = self.deli.join(_images)
-            if get_torch_version_tuple() > (1, 6, 0):
-                # all gather across all processes
-                _filenames = self.deli.join(idist.all_gather(_filenames))
-            else:
-                raise RuntimeError("MetricsSaver can not save metric details in distributed mode with PyTorch < 1.7.0.")
-            _images = _filenames.split(self.deli)
+        # all gather file names across ranks
+        _images = string_list_all_gather(strings=self._filenames) if ws > 1 else self._filenames
 
         # only save metrics to file in specified rank
         if idist.get_rank() == self.save_rank:
diff --git a/monai/handlers/roc_auc.py b/monai/handlers/roc_auc.py
index dbca70bf25..2273b9ee89 100644
--- a/monai/handlers/roc_auc.py
+++ b/monai/handlers/roc_auc.py
@@ -9,13 +9,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Callable, Optional, Union
+from typing import Any, Callable, Optional, Union
 
 import torch
 
+from monai.handlers.utils import evenly_divisible_all_gather
 from monai.metrics import compute_roc_auc
 from monai.utils import Average, exact_version, optional_import
 
+idist, _ = optional_import("ignite", "0.4.2", exact_version, "distributed")
 EpochMetric, _ = optional_import("ignite.metrics", "0.4.2", exact_version, "EpochMetric")
 
 
@@ -71,9 +73,32 @@ def _compute_fn(pred, label):
                 average=Average(average),
             )
 
+        self._is_reduced: bool = False
         super().__init__(
             compute_fn=_compute_fn,
             output_transform=output_transform,
             check_compute_fn=False,
             device=device,
         )
+
+    def compute(self) -> Any:
+        _prediction_tensor = torch.cat(self._predictions, dim=0)
+        _target_tensor = torch.cat(self._targets, dim=0)
+
+        ws = idist.get_world_size()
+        if ws > 1 and not self._is_reduced:
+            # All gather across all processes
+            _prediction_tensor = evenly_divisible_all_gather(_prediction_tensor)
+            _target_tensor = evenly_divisible_all_gather(_target_tensor)
+        self._is_reduced = True
+
+        result: torch.Tensor = torch.zeros(1)
+        if idist.get_rank() == 0:
+            # Run compute_fn on zero rank only
+            result = self.compute_fn(_prediction_tensor, _target_tensor)
+
+        if ws > 1:
+            # broadcast result to all processes
+            result = idist.broadcast(result, src=0)
+
+        return result.item() if torch.is_tensor(result) else result
diff --git a/monai/handlers/segmentation_saver.py b/monai/handlers/segmentation_saver.py
index c712ce9a9e..a46918b893 100644
--- a/monai/handlers/segmentation_saver.py
+++ b/monai/handlers/segmentation_saver.py
@@ -14,7 +14,8 @@
 
 import numpy as np
 
-from monai.data import NiftiSaver, PNGSaver
+from monai.config import DtypeLike
+from monai.transforms import SaveImage
 from monai.utils import GridSampleMode, GridSamplePadMode, InterpolateMode, exact_version, optional_import
 
 Events, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Events")
@@ -38,8 +39,8 @@ def __init__(
         mode: Union[GridSampleMode, InterpolateMode, str] = "nearest",
         padding_mode: Union[GridSamplePadMode, str] = GridSamplePadMode.BORDER,
         scale: Optional[int] = None,
-        dtype: Optional[np.dtype] = np.float64,
-        output_dtype: Optional[np.dtype] = np.float32,
+        dtype: DtypeLike = np.float64,
+        output_dtype: DtypeLike = np.float32,
         batch_transform: Callable = lambda x: x,
         output_transform: Callable = lambda x: x,
         name: Optional[str] = None,
@@ -47,9 +48,11 @@ def __init__(
         """
         Args:
             output_dir: output image directory.
-            output_postfix: a string appended to all output file names.
-            output_ext: output file extension name.
+            output_postfix: a string appended to all output file names, default to `seg`.
+            output_ext: output file extension name, available extensions: `.nii.gz`, `.nii`, `.png`.
             resample: whether to resample before saving the data array.
+                if saving PNG format image, based on the `spatial_shape` from metadata.
+                if saving NIfTI format image, based on the `original_affine` from metadata.
             mode: This option is used when ``resample = True``. Defaults to ``"nearest"``.
 
                 - NIfTI files {``"bilinear"``, ``"nearest"``}
@@ -71,8 +74,8 @@ def __init__(
                 [0, 255] (uint8) or [0, 65535] (uint16). Default is None to disable scaling.
                 It's used for PNG format only.
             dtype: data type for resampling computation. Defaults to ``np.float64`` for best precision.
-                If None, use the data type of input data. To be compatible with other modules,
-                the output data type is always ``np.float32``, it's used for Nifti format only.
+                If None, use the data type of input data.
+                It's used for Nifti format only.
             output_dtype: data type for saving data. Defaults to ``np.float32``, it's used for Nifti format only.
             batch_transform: a callable that is used to transform the
                 ignite.engine.batch into expected format to extract the meta_data dictionary.
@@ -83,27 +86,18 @@ def __init__(
             name: identifier of logging.logger to use, defaulting to `engine.logger`.
 
         """
-        self.saver: Union[NiftiSaver, PNGSaver]
-        if output_ext in (".nii.gz", ".nii"):
-            self.saver = NiftiSaver(
-                output_dir=output_dir,
-                output_postfix=output_postfix,
-                output_ext=output_ext,
-                resample=resample,
-                mode=GridSampleMode(mode),
-                padding_mode=padding_mode,
-                dtype=dtype,
-                output_dtype=output_dtype,
-            )
-        elif output_ext == ".png":
-            self.saver = PNGSaver(
-                output_dir=output_dir,
-                output_postfix=output_postfix,
-                output_ext=output_ext,
-                resample=resample,
-                mode=InterpolateMode(mode),
-                scale=scale,
-            )
+        self._saver = SaveImage(
+            output_dir=output_dir,
+            output_postfix=output_postfix,
+            output_ext=output_ext,
+            resample=resample,
+            mode=mode,
+            padding_mode=padding_mode,
+            scale=scale,
+            dtype=dtype,
+            output_dtype=output_dtype,
+            save_batch=True,
+        )
         self.batch_transform = batch_transform
         self.output_transform = output_transform
 
@@ -130,5 +124,5 @@ def __call__(self, engine: Engine) -> None:
         """
         meta_data = self.batch_transform(engine.state.batch)
         engine_output = self.output_transform(engine.state.output)
-        self.saver.save_batch(engine_output, meta_data)
+        self._saver(engine_output, meta_data)
         self.logger.info("saved all the model outputs into files.")
diff --git a/monai/handlers/stats_handler.py b/monai/handlers/stats_handler.py
index 007fbed413..24d844569f 100644
--- a/monai/handlers/stats_handler.py
+++ b/monai/handlers/stats_handler.py
@@ -196,10 +196,12 @@ def _default_iteration_print(self, engine: Engine) -> None:
                         " {}:{}".format(name, type(value))
                     )
                     continue  # not printing multi dimensional output
-                out_str += self.key_var_format.format(name, value.item() if torch.is_tensor(value) else value)
+                out_str += self.key_var_format.format(name, value.item() if isinstance(value, torch.Tensor) else value)
         else:
             if is_scalar(loss):  # not printing multi dimensional output
-                out_str += self.key_var_format.format(self.tag_name, loss.item() if torch.is_tensor(loss) else loss)
+                out_str += self.key_var_format.format(
+                    self.tag_name, loss.item() if isinstance(loss, torch.Tensor) else loss
+                )
             else:
                 warnings.warn(
                     "ignoring non-scalar output in StatsHandler,"
diff --git a/monai/handlers/tensorboard_handlers.py b/monai/handlers/tensorboard_handlers.py
index 15fa6a5eed..4ee88bcfc9 100644
--- a/monai/handlers/tensorboard_handlers.py
+++ b/monai/handlers/tensorboard_handlers.py
@@ -29,7 +29,38 @@
 DEFAULT_TAG = "Loss"
 
 
-class TensorBoardStatsHandler:
+class TensorBoardHandler:
+    """
+    Base class for the handlers to write data into TensorBoard.
+
+    Args:
+        summary_writer: user can specify TensorBoard SummaryWriter,
+            default to create a new writer.
+        log_dir: if using default SummaryWriter, write logs to this directory, default is `./runs`.
+
+    """
+
+    def __init__(self, summary_writer: Optional[SummaryWriter] = None, log_dir: str = "./runs"):
+        if summary_writer is None:
+            self._writer = SummaryWriter(log_dir=log_dir)
+            self.internal_writer = True
+        else:
+            self._writer = summary_writer
+            self.internal_writer = False
+
+    def attach(self, engine: Engine) -> None:
+        raise NotImplementedError(f"Subclass {self.__class__.__name__} must implement this method.")
+
+    def close(self):
+        """
+        Close the summary writer if created in this TensorBoard handler.
+
+        """
+        if self.internal_writer:
+            self._writer.close()
+
+
+class TensorBoardStatsHandler(TensorBoardHandler):
     """
     TensorBoardStatsHandler defines a set of Ignite Event-handlers for all the TensorBoard logics.
     It's can be used for any Ignite Engine(trainer, validator and evaluator).
@@ -71,7 +102,7 @@ def __init__(
                 when plotting epoch vs metric curves.
             tag_name: when iteration output is a scalar, tag_name is used to plot, defaults to ``'Loss'``.
         """
-        self._writer = SummaryWriter(log_dir=log_dir) if summary_writer is None else summary_writer
+        super().__init__(summary_writer=summary_writer, log_dir=log_dir)
         self.epoch_event_writer = epoch_event_writer
         self.iteration_event_writer = iteration_event_writer
         self.output_transform = output_transform
@@ -159,9 +190,13 @@ def _default_iteration_writer(self, engine: Engine, writer: SummaryWriter) -> No
                         " {}:{}".format(name, type(value))
                     )
                     continue  # not plot multi dimensional output
-                writer.add_scalar(name, value.item() if torch.is_tensor(value) else value, engine.state.iteration)
+                writer.add_scalar(
+                    name, value.item() if isinstance(value, torch.Tensor) else value, engine.state.iteration
+                )
         elif is_scalar(loss):  # not printing multi dimensional output
-            writer.add_scalar(self.tag_name, loss.item() if torch.is_tensor(loss) else loss, engine.state.iteration)
+            writer.add_scalar(
+                self.tag_name, loss.item() if isinstance(loss, torch.Tensor) else loss, engine.state.iteration
+            )
         else:
             warnings.warn(
                 "ignoring non-scalar output in TensorBoardStatsHandler,"
@@ -172,7 +207,7 @@ def _default_iteration_writer(self, engine: Engine, writer: SummaryWriter) -> No
         writer.flush()
 
 
-class TensorBoardImageHandler:
+class TensorBoardImageHandler(TensorBoardHandler):
     """
     TensorBoardImageHandler is an Ignite Event handler that can visualize images, labels and outputs as 2D/3D images.
     2D output (shape in Batch, channel, H, W) will be shown as simple image using the first element in the batch,
@@ -225,7 +260,7 @@ def __init__(
             max_channels: number of channels to plot.
             max_frames: number of frames for 2D-t plot.
         """
-        self._writer = SummaryWriter(log_dir=log_dir) if summary_writer is None else summary_writer
+        super().__init__(summary_writer=summary_writer, log_dir=log_dir)
         self.interval = interval
         self.epoch_level = epoch_level
         self.batch_transform = batch_transform
@@ -261,7 +296,7 @@ def __call__(self, engine: Engine) -> None:
         """
         step = self.global_iter_transform(engine.state.epoch if self.epoch_level else engine.state.iteration)
         show_images = self.batch_transform(engine.state.batch)[0]
-        if torch.is_tensor(show_images):
+        if isinstance(show_images, torch.Tensor):
             show_images = show_images.detach().cpu().numpy()
         if show_images is not None:
             if not isinstance(show_images, np.ndarray):
@@ -274,7 +309,7 @@ def __call__(self, engine: Engine) -> None:
             )
 
         show_labels = self.batch_transform(engine.state.batch)[1]
-        if torch.is_tensor(show_labels):
+        if isinstance(show_labels, torch.Tensor):
             show_labels = show_labels.detach().cpu().numpy()
         if show_labels is not None:
             if not isinstance(show_labels, np.ndarray):
@@ -287,7 +322,7 @@ def __call__(self, engine: Engine) -> None:
             )
 
         show_outputs = self.output_transform(engine.state.output)
-        if torch.is_tensor(show_outputs):
+        if isinstance(show_outputs, torch.Tensor):
             show_outputs = show_outputs.detach().cpu().numpy()
         if show_outputs is not None:
             if not isinstance(show_outputs, np.ndarray):
diff --git a/monai/handlers/utils.py b/monai/handlers/utils.py
index ef652efe0a..3e36af0652 100644
--- a/monai/handlers/utils.py
+++ b/monai/handlers/utils.py
@@ -11,12 +11,12 @@
 
 import os
 from collections import OrderedDict
-from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Sequence, Union
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Sequence, Union
 
 import numpy as np
 import torch
 
-from monai.utils import ensure_tuple, exact_version, optional_import
+from monai.utils import ensure_tuple, exact_version, get_torch_version_tuple, optional_import
 
 idist, _ = optional_import("ignite", "0.4.2", exact_version, "distributed")
 if TYPE_CHECKING:
@@ -28,6 +28,7 @@
     "stopping_fn_from_metric",
     "stopping_fn_from_loss",
     "evenly_divisible_all_gather",
+    "string_list_all_gather",
     "write_metrics_reports",
 ]
 
@@ -61,8 +62,11 @@ def evenly_divisible_all_gather(data: torch.Tensor) -> torch.Tensor:
     Args:
         data: source tensor to pad and execute all_gather in distributed data parallel.
 
+    Note:
+        The input data on different ranks must have exactly same `dtype`.
+
     """
-    if not torch.is_tensor(data):
+    if not isinstance(data, torch.Tensor):
         raise ValueError("input data must be PyTorch Tensor.")
 
     if idist.get_world_size() <= 1:
@@ -81,6 +85,29 @@ def evenly_divisible_all_gather(data: torch.Tensor) -> torch.Tensor:
     return torch.cat([data[i * max_len : i * max_len + l, ...] for i, l in enumerate(all_lens)], dim=0)
 
 
+def string_list_all_gather(strings: List[str], delimiter: str = "\t") -> List[str]:
+    """
+    Utility function for distributed data parallel to all gather a list of strings.
+
+    Args:
+        strings: a list of strings to all gather.
+        delimiter: use the delimiter to join the string list to be a long string,
+            then all gather across ranks and split to a list. default to "\t".
+
+    """
+    if idist.get_world_size() <= 1:
+        return strings
+
+    _joined = delimiter.join(strings)
+    if get_torch_version_tuple() > (1, 6, 0):
+        # all gather across all ranks
+        _joined = delimiter.join(idist.all_gather(_joined))
+    else:
+        raise RuntimeError("string all_gather can not be supported in PyTorch < 1.7.0.")
+
+    return _joined.split(delimiter)
+
+
 def write_metrics_reports(
     save_dir: str,
     images: Optional[Sequence[str]],
@@ -101,7 +128,7 @@ def write_metrics_reports(
         images: name or path of every input image corresponding to the metric_details data.
             if None, will use index number as the filename of every input image.
         metrics: a dictionary of (metric name, metric value) pairs.
-        metric_details: a dictionary of (metric name, metric raw values) pairs,
+        metric_details: a dictionary of (metric name, metric raw values) pairs, usually, it comes from metrics computation,
             for example, the raw value can be the mean_dice of every channel of every input image.
         summary_ops: expected computation operations to generate the summary report.
             it can be: None, "*" or list of strings.
@@ -110,7 +137,7 @@ def write_metrics_reports(
             list of strings - generate summary report for every metric_details with specified operations, they
             should be within this list: [`mean`, `median`, `max`, `min`, `90percent`, `std`].
             default to None.
-        deli: the delimiter charactor in the file, default to "\t".
+        deli: the delimiter character in the file, default to "\t".
         output_type: expected output file type, supported types: ["csv"], default to "csv".
 
     """
@@ -127,7 +154,7 @@ def write_metrics_reports(
 
     if metric_details is not None and len(metric_details) > 0:
         for k, v in metric_details.items():
-            if torch.is_tensor(v):
+            if isinstance(v, torch.Tensor):
                 v = v.cpu().numpy()
             if v.ndim == 0:
                 # reshape to [1, 1] if no batch and class dims
@@ -162,5 +189,5 @@ def write_metrics_reports(
 
                 with open(os.path.join(save_dir, f"{k}_summary.csv"), "w") as f:
                     f.write(f"class{deli}{deli.join(ops)}\n")
-                    for i, c in enumerate(v.transpose()):
+                    for i, c in enumerate(np.transpose(v)):
                         f.write(f"{class_labels[i]}{deli}{deli.join([f'{supported_ops[k](c):.4f}' for k in ops])}\n")
diff --git a/monai/losses/__init__.py b/monai/losses/__init__.py
index 591fb08f7b..b9146a6962 100644
--- a/monai/losses/__init__.py
+++ b/monai/losses/__init__.py
@@ -23,4 +23,5 @@
 )
 from .focal_loss import FocalLoss
 from .image_dissimilarity import GlobalMutualInformationLoss, LocalNormalizedCrossCorrelationLoss
+from .multi_scale import MultiScaleLoss
 from .tversky import TverskyLoss
diff --git a/monai/losses/dice.py b/monai/losses/dice.py
index f14aa6955f..c284660cc6 100644
--- a/monai/losses/dice.py
+++ b/monai/losses/dice.py
@@ -508,7 +508,7 @@ def wasserstein_distance_map(self, flat_proba: torch.Tensor, flat_target: torch.
             flat_target: the target tensor.
         """
         # Turn the distance matrix to a map of identical matrix
-        m = torch.clone(self.m).to(flat_proba.device)
+        m = torch.clone(torch.as_tensor(self.m)).to(flat_proba.device)
         m_extended = torch.unsqueeze(m, dim=0)
         m_extended = torch.unsqueeze(m_extended, dim=3)
         m_extended = m_extended.expand((flat_proba.size(0), m_extended.size(1), m_extended.size(2), flat_proba.size(2)))
diff --git a/monai/losses/multi_scale.py b/monai/losses/multi_scale.py
new file mode 100644
index 0000000000..5a17bc2d07
--- /dev/null
+++ b/monai/losses/multi_scale.py
@@ -0,0 +1,100 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List, Optional, Union
+
+import torch
+from torch.nn.modules.loss import _Loss
+
+from monai.networks.layers import gaussian_1d, separable_filtering
+from monai.utils import LossReduction
+
+
+def make_gaussian_kernel(sigma: int) -> torch.Tensor:
+    if sigma <= 0:
+        raise ValueError(f"expecting positive sigma, got sigma={sigma}")
+    kernel = gaussian_1d(sigma=torch.tensor(sigma), truncated=3, approx="sampled", normalize=False)
+    return kernel
+
+
+def make_cauchy_kernel(sigma: int) -> torch.Tensor:
+    if sigma <= 0:
+        raise ValueError(f"expecting positive sigma, got sigma={sigma}")
+    tail = int(sigma * 5)
+    k = torch.tensor([((x / sigma) ** 2 + 1) for x in range(-tail, tail + 1)])
+    k = torch.reciprocal(k)
+    k = k / torch.sum(k)
+    return k
+
+
+kernel_fn_dict = {
+    "gaussian": make_gaussian_kernel,
+    "cauchy": make_cauchy_kernel,
+}
+
+
+class MultiScaleLoss(_Loss):
+    """
+    This is a wrapper class.
+    It smooths the input and target at different scales before passing them into the wrapped loss function.
+
+    Adapted from:
+        DeepReg (https://github.com/DeepRegNet/DeepReg)
+    """
+
+    def __init__(
+        self,
+        loss: _Loss,
+        scales: Optional[List] = None,
+        kernel: str = "gaussian",
+        reduction: Union[LossReduction, str] = LossReduction.MEAN,
+    ) -> None:
+        """
+        Args:
+            loss: loss function to be wrapped
+            scales: list of scalars or None, if None, do not apply any scaling.
+            kernel: gaussian or cauchy.
+        """
+        super(MultiScaleLoss, self).__init__(reduction=LossReduction(reduction).value)
+        if kernel not in kernel_fn_dict.keys():
+            raise ValueError(f"got unsupported kernel type: {kernel}", "only support gaussian and cauchy")
+        self.kernel_fn = kernel_fn_dict[kernel]
+        self.loss = loss
+        self.scales = scales
+
+    def forward(self, y_true: torch.Tensor, y_pred: torch.Tensor) -> torch.Tensor:
+        if self.scales is None:
+            loss: torch.Tensor = self.loss(y_pred, y_true)
+        else:
+            loss_list = []
+            for s in self.scales:
+                if s == 0:
+                    # no smoothing
+                    loss_list.append(self.loss(y_pred, y_true))
+                else:
+                    loss_list.append(
+                        self.loss(
+                            separable_filtering(y_pred, [self.kernel_fn(s)] * (y_true.ndim - 2)),
+                            separable_filtering(y_true, [self.kernel_fn(s)] * (y_true.ndim - 2)),
+                        )
+                    )
+            loss = torch.stack(loss_list, dim=0)
+
+        if self.reduction == LossReduction.MEAN.value:
+            loss = torch.mean(loss)  # the batch and channel average
+        elif self.reduction == LossReduction.SUM.value:
+            loss = torch.sum(loss)  # sum over the batch and channel dims
+        elif self.reduction == LossReduction.NONE.value:
+            pass  # returns [N, n_classes] losses
+        else:
+            raise ValueError(f'Unsupported reduction: {self.reduction}, available options are ["mean", "sum", "none"].')
+
+        return loss
diff --git a/monai/metrics/hausdorff_distance.py b/monai/metrics/hausdorff_distance.py
index 8ecc19ec46..6570ace800 100644
--- a/monai/metrics/hausdorff_distance.py
+++ b/monai/metrics/hausdorff_distance.py
@@ -127,9 +127,10 @@ def compute_hausdorff_distance(
             y_pred=y_pred,
             y=y,
         )
-
-    y = y.float()
-    y_pred = y_pred.float()
+    if isinstance(y, torch.Tensor):
+        y = y.float()
+    if isinstance(y_pred, torch.Tensor):
+        y_pred = y_pred.float()
 
     if y.shape != y_pred.shape:
         raise ValueError("y_pred and y should have same shapes.")
diff --git a/monai/metrics/rocauc.py b/monai/metrics/rocauc.py
index 9f081d1698..80a6671dfa 100644
--- a/monai/metrics/rocauc.py
+++ b/monai/metrics/rocauc.py
@@ -10,7 +10,7 @@
 # limitations under the License.
 
 import warnings
-from typing import Callable, List, Optional, Union, cast
+from typing import Callable, Optional, Union, cast
 
 import numpy as np
 import torch
@@ -57,7 +57,7 @@ def compute_roc_auc(
     softmax: bool = False,
     other_act: Optional[Callable] = None,
     average: Union[Average, str] = Average.MACRO,
-) -> Union[np.ndarray, List[float], float]:
+):
     """Computes Area Under the Receiver Operating Characteristic Curve (ROC AUC). Referring to:
     `sklearn.metrics.roc_auc_score <https://scikit-learn.org/stable/modules/generated/
     sklearn.metrics.roc_auc_score.html#sklearn.metrics.roc_auc_score>`_.
diff --git a/monai/metrics/surface_distance.py b/monai/metrics/surface_distance.py
index 9e2f130bd2..b605fdb88f 100644
--- a/monai/metrics/surface_distance.py
+++ b/monai/metrics/surface_distance.py
@@ -120,8 +120,10 @@ def compute_average_surface_distance(
             y=y,
         )
 
-    y = y.float()
-    y_pred = y_pred.float()
+    if isinstance(y, torch.Tensor):
+        y = y.float()
+    if isinstance(y_pred, torch.Tensor):
+        y_pred = y_pred.float()
 
     if y.shape != y_pred.shape:
         raise ValueError("y_pred and y should have same shapes.")
@@ -135,7 +137,7 @@ def compute_average_surface_distance(
         if surface_distance.shape == (0,):
             avg_surface_distance = np.nan
         else:
-            avg_surface_distance = surface_distance.mean()
+            avg_surface_distance = surface_distance.mean()  # type: ignore
         if not symmetric:
             asd[b, c] = avg_surface_distance
         else:
@@ -143,7 +145,7 @@ def compute_average_surface_distance(
             if surface_distance_2.shape == (0,):
                 avg_surface_distance_2 = np.nan
             else:
-                avg_surface_distance_2 = surface_distance_2.mean()
+                avg_surface_distance_2 = surface_distance_2.mean()  # type: ignore
             asd[b, c] = np.mean((avg_surface_distance, avg_surface_distance_2))
 
     return torch.from_numpy(asd)
diff --git a/monai/metrics/utils.py b/monai/metrics/utils.py
index cc7049ff81..0a254d9901 100644
--- a/monai/metrics/utils.py
+++ b/monai/metrics/utils.py
@@ -26,8 +26,8 @@
 
 
 def ignore_background(
-    y_pred: torch.Tensor,
-    y: torch.Tensor,
+    y_pred: Union[np.ndarray, torch.Tensor],
+    y: Union[np.ndarray, torch.Tensor],
 ):
     """
     This function is used to remove background (the first channel) for `y_pred` and `y`.
@@ -138,9 +138,9 @@ def get_mask_edges(
     """
 
     # Get both labelfields as np arrays
-    if torch.is_tensor(seg_pred):
+    if isinstance(seg_pred, torch.Tensor):
         seg_pred = seg_pred.detach().cpu().numpy()
-    if torch.is_tensor(seg_gt):
+    if isinstance(seg_gt, torch.Tensor):
         seg_gt = seg_gt.detach().cpu().numpy()
 
     if seg_pred.shape != seg_gt.shape:
@@ -157,7 +157,7 @@ def get_mask_edges(
             return (np.zeros_like(seg_pred), np.zeros_like(seg_gt))
 
         seg_pred, seg_gt = np.expand_dims(seg_pred, 0), np.expand_dims(seg_gt, 0)
-        box_start, box_end = generate_spatial_bounding_box(seg_pred | seg_gt)
+        box_start, box_end = generate_spatial_bounding_box(np.asarray(seg_pred | seg_gt))
         cropper = SpatialCrop(roi_start=box_start, roi_end=box_end)
         seg_pred, seg_gt = np.squeeze(cropper(seg_pred)), np.squeeze(cropper(seg_gt))
 
@@ -192,7 +192,7 @@ def get_surface_distance(
     else:
         if not np.any(seg_pred):
             dis = np.inf * np.ones_like(seg_gt)
-            return dis[seg_gt]
+            return np.asarray(dis[seg_gt])
         if distance_metric == "euclidean":
             dis = distance_transform_edt(~seg_gt)
         elif distance_metric in ["chessboard", "taxicab"]:
@@ -200,4 +200,4 @@ def get_surface_distance(
         else:
             raise ValueError(f"distance_metric {distance_metric} is not implemented.")
 
-    return dis[seg_pred]
+    return np.asarray(dis[seg_pred])
diff --git a/monai/networks/blocks/__init__.py b/monai/networks/blocks/__init__.py
index 8ac06f8776..4a2e31928e 100644
--- a/monai/networks/blocks/__init__.py
+++ b/monai/networks/blocks/__init__.py
@@ -27,4 +27,4 @@
     SEResNeXtBottleneck,
 )
 from .upsample import SubpixelUpsample, Subpixelupsample, SubpixelUpSample, Upsample, UpSample
-from .warp import Warp
+from .warp import DVF2DDF, Warp
diff --git a/monai/networks/blocks/warp.py b/monai/networks/blocks/warp.py
index 60e23f6750..eb4c09fa72 100644
--- a/monai/networks/blocks/warp.py
+++ b/monai/networks/blocks/warp.py
@@ -62,7 +62,7 @@ def forward(self, image: torch.Tensor, ddf: torch.Tensor) -> torch.Tensor:
         """
         Args:
             image: Tensor in shape (batch, num_channels, H, W[, D])
-            ddf: Tensor in the same spatial size as image, in shape (batch, spatial_dims, H, W[, D])
+            ddf: Tensor in the same spatial size as image, in shape (batch, ``spatial_dims``, H, W[, D])
 
         Returns:
             warped_image in the same shape as image (batch, num_channels, H, W[, D])
@@ -111,3 +111,40 @@ def forward(self, image: torch.Tensor, ddf: torch.Tensor) -> torch.Tensor:
             )
 
         return warped_image
+
+
+class DVF2DDF(nn.Module):
+    """
+    Layer calculates a dense velocity field (DVF) from a dense displacement field (DDF)
+    with scaling and squaring.
+
+    Adapted from:
+        DeepReg (https://github.com/DeepRegNet/DeepReg)
+
+    """
+
+    def __init__(
+        self,
+        spatial_dims: int,
+        num_steps: int = 7,
+        mode: int = 1,
+        padding_mode: Optional[Union[GridSamplePadMode, str]] = GridSamplePadMode.ZEROS,
+    ):
+        super(DVF2DDF, self).__init__()
+        if num_steps <= 0:
+            raise ValueError(f"expecting positive num_steps, got {num_steps}")
+        self.num_steps = num_steps
+        self.warp_layer = Warp(spatial_dims=spatial_dims, mode=mode, padding_mode=padding_mode)
+
+    def forward(self, dvf):
+        """
+        Args:
+            dvf: dvf to be transformed, in shape (batch, ``spatial_dims``, H, W[,D])
+
+        Returns:
+
+        """
+        ddf: torch.Tensor = dvf / (2 ** self.num_steps)
+        for _ in range(self.num_steps):
+            ddf = ddf + self.warp_layer(image=ddf, ddf=ddf)
+        return ddf
diff --git a/monai/networks/layers/convutils.py b/monai/networks/layers/convutils.py
index c4f798699c..994ca05b85 100644
--- a/monai/networks/layers/convutils.py
+++ b/monai/networks/layers/convutils.py
@@ -57,7 +57,7 @@ def stride_minus_kernel_padding(
 
 
 def calculate_out_shape(
-    in_shape: Union[Sequence[int], int],
+    in_shape: Union[Sequence[int], int, np.ndarray],
     kernel_size: Union[Sequence[int], int],
     stride: Union[Sequence[int], int],
     padding: Union[Sequence[int], int],
@@ -104,7 +104,7 @@ def gaussian_1d(
         1D torch tensor
 
     """
-    sigma = torch.as_tensor(sigma, dtype=torch.float, device=sigma.device if torch.is_tensor(sigma) else None)
+    sigma = torch.as_tensor(sigma, dtype=torch.float, device=sigma.device if isinstance(sigma, torch.Tensor) else None)
     device = sigma.device
     if truncated <= 0.0:
         raise ValueError(f"truncated must be positive, got {truncated}.")
@@ -149,7 +149,7 @@ def polyval(coef, x) -> torch.Tensor:
     Returns:
         1D torch tensor
     """
-    device = x.device if torch.is_tensor(x) else None
+    device = x.device if isinstance(x, torch.Tensor) else None
     coef = torch.as_tensor(coef, dtype=torch.float, device=device)
     if coef.ndim == 0 or (len(coef) < 1):
         return torch.zeros(x.shape)
@@ -161,7 +161,7 @@ def polyval(coef, x) -> torch.Tensor:
 
 
 def _modified_bessel_0(x: torch.Tensor) -> torch.Tensor:
-    x = torch.as_tensor(x, dtype=torch.float, device=x.device if torch.is_tensor(x) else None)
+    x = torch.as_tensor(x, dtype=torch.float, device=x.device if isinstance(x, torch.Tensor) else None)
     if torch.abs(x) < 3.75:
         y = x * x / 14.0625
         return polyval([0.45813e-2, 0.360768e-1, 0.2659732, 1.2067492, 3.0899424, 3.5156229, 1.0], y)
@@ -182,7 +182,7 @@ def _modified_bessel_0(x: torch.Tensor) -> torch.Tensor:
 
 
 def _modified_bessel_1(x: torch.Tensor) -> torch.Tensor:
-    x = torch.as_tensor(x, dtype=torch.float, device=x.device if torch.is_tensor(x) else None)
+    x = torch.as_tensor(x, dtype=torch.float, device=x.device if isinstance(x, torch.Tensor) else None)
     if torch.abs(x) < 3.75:
         y = x * x / 14.0625
         _coef = [0.32411e-3, 0.301532e-2, 0.2658733e-1, 0.15084934, 0.51498869, 0.87890594, 0.5]
@@ -207,7 +207,7 @@ def _modified_bessel_1(x: torch.Tensor) -> torch.Tensor:
 def _modified_bessel_i(n: int, x: torch.Tensor) -> torch.Tensor:
     if n < 2:
         raise ValueError(f"n must be greater than 1, got n={n}.")
-    x = torch.as_tensor(x, dtype=torch.float, device=x.device if torch.is_tensor(x) else None)
+    x = torch.as_tensor(x, dtype=torch.float, device=x.device if isinstance(x, torch.Tensor) else None)
     if x == 0.0:
         return x
     device = x.device
diff --git a/monai/networks/layers/filtering.py b/monai/networks/layers/filtering.py
index 83a33bc609..1bec725c7e 100644
--- a/monai/networks/layers/filtering.py
+++ b/monai/networks/layers/filtering.py
@@ -62,7 +62,7 @@ class PHLFilter(torch.autograd.Function):
     """
     Filters input based on arbitrary feature vectors. Uses a permutohedral
     lattice data structure to efficiently approximate n-dimensional gaussian
-    filtering. Complexity is broadly independant of kernel size. Most applicable
+    filtering. Complexity is broadly independent of kernel size. Most applicable
     to higher filter dimensions and larger kernel sizes.
 
     See:
diff --git a/monai/networks/layers/simplelayers.py b/monai/networks/layers/simplelayers.py
index 285b0d629f..f560526db8 100644
--- a/monai/networks/layers/simplelayers.py
+++ b/monai/networks/layers/simplelayers.py
@@ -182,12 +182,12 @@ def separable_filtering(
         TypeError: When ``x`` is not a ``torch.Tensor``.
     """
 
-    if not torch.is_tensor(x):
+    if not isinstance(x, torch.Tensor):
         raise TypeError(f"x must be a torch.Tensor but is {type(x).__name__}.")
 
     spatial_dims = len(x.shape) - 2
     _kernels = [
-        torch.as_tensor(s, dtype=torch.float, device=s.device if torch.is_tensor(s) else None)
+        torch.as_tensor(s, dtype=torch.float, device=s.device if isinstance(s, torch.Tensor) else None)
         for s in ensure_tuple_rep(kernels, spatial_dims)
     ]
     _paddings = [cast(int, (same_padding(k.shape[0]))) for k in _kernels]
@@ -251,7 +251,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor:
         """
 
         # Make input a real tensor on the CPU
-        x = torch.as_tensor(x, device=x.device if torch.is_tensor(x) else None)
+        x = torch.as_tensor(x, device=x.device if isinstance(x, torch.Tensor) else None)
         if torch.is_complex(x):
             raise ValueError("x must be real.")
         else:
@@ -317,7 +317,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor:
         """
 
         # Make input a real tensor
-        x = torch.as_tensor(x, device=x.device if torch.is_tensor(x) else None)
+        x = torch.as_tensor(x, device=x.device if isinstance(x, torch.Tensor) else None)
         if torch.is_complex(x):
             raise ValueError("x must be real.")
         x = x.to(dtype=torch.float)
@@ -384,7 +384,7 @@ def __init__(
         super().__init__()
         self.sigma = [
             torch.nn.Parameter(
-                torch.as_tensor(s, dtype=torch.float, device=s.device if torch.is_tensor(s) else None),
+                torch.as_tensor(s, dtype=torch.float, device=s.device if isinstance(s, torch.Tensor) else None),
                 requires_grad=requires_grad,
             )
             for s in ensure_tuple_rep(sigma, int(spatial_dims))
diff --git a/monai/networks/layers/spatial_transforms.py b/monai/networks/layers/spatial_transforms.py
index c0f22502c8..175fd05694 100644
--- a/monai/networks/layers/spatial_transforms.py
+++ b/monai/networks/layers/spatial_transforms.py
@@ -487,7 +487,7 @@ def forward(
 
         """
         # validate `theta`
-        if not torch.is_tensor(theta):
+        if not isinstance(theta, torch.Tensor):
             raise TypeError(f"theta must be torch.Tensor but is {type(theta).__name__}.")
         if theta.dim() not in (2, 3):
             raise ValueError(f"theta must be Nxdxd or dxd, got {theta.shape}.")
@@ -504,7 +504,7 @@ def forward(
             raise ValueError(f"theta must be Nx3x3 or Nx4x4, got {theta.shape}.")
 
         # validate `src`
-        if not torch.is_tensor(src):
+        if not isinstance(src, torch.Tensor):
             raise TypeError(f"src must be torch.Tensor but is {type(src).__name__}.")
         sr = src.dim() - 2  # input spatial rank
         if sr not in (2, 3):
diff --git a/monai/networks/nets/ahnet.py b/monai/networks/nets/ahnet.py
index 847993bd44..3321001af0 100644
--- a/monai/networks/nets/ahnet.py
+++ b/monai/networks/nets/ahnet.py
@@ -434,8 +434,7 @@ def __init__(
         self.dense4 = DenseBlock(spatial_dims, ndenselayer, num_init_features, densebn, densegrowth, 0.0)
         noutdense4 = num_init_features + densegrowth * ndenselayer
 
-        if psp_block_num > 0:
-            self.psp = PSP(spatial_dims, psp_block_num, noutdense4, upsample_mode)
+        self.psp = PSP(spatial_dims, psp_block_num, noutdense4, upsample_mode)
         self.final = Final(spatial_dims, psp_block_num + noutdense4, out_channels, upsample_mode)
 
         # Initialise parameters
@@ -511,7 +510,7 @@ def forward(self, x):
 
         sum4 = self.up3(d3) + conv_x
         d4 = self.dense4(sum4)
-        if self.psp_block_num > 0 and self.psp is not None:
+        if self.psp_block_num > 0:
             psp = self.psp(d4)
             x = torch.cat((psp, d4), dim=1)
         else:
diff --git a/monai/networks/nets/dynunet.py b/monai/networks/nets/dynunet.py
index ba88c35f8d..7d0b3bff79 100644
--- a/monai/networks/nets/dynunet.py
+++ b/monai/networks/nets/dynunet.py
@@ -14,6 +14,7 @@
 
 import torch
 import torch.nn as nn
+from torch.nn.functional import interpolate
 
 from monai.networks.blocks.dynunet_block import UnetBasicBlock, UnetOutBlock, UnetResBlock, UnetUpBlock
 
@@ -80,8 +81,22 @@ class DynUNet(nn.Module):
         upsample_kernel_size: convolution kernel size for transposed convolution layers.
         norm_name: [``"batch"``, ``"instance"``, ``"group"``]
             feature normalization type and arguments.
+        deep_supervision: whether to add deep supervision head before output. Defaults to ``False``.
+            If ``True``, in training mode, the forward function will output not only the last feature
+            map, but also the previous feature maps that come from the intermediate up sample layers.
+            In order to unify the return type (the restriction of TorchScript), all intermediate
+            feature maps are interpolated into the same size as the last feature map and stacked together
+            (with a new dimension in the first axis)into one single tensor.
+            For instance, if there are three feature maps with shapes: (1, 2, 32, 24), (1, 2, 16, 12) and
+            (1, 2, 8, 6). The last two will be interpolated into (1, 2, 32, 24), and the stacked tensor
+            will has the shape (1, 3, 2, 8, 6).
+            When calculating the loss, you can use torch.unbind to get all feature maps can compute the loss
+            one by one with the groud truth, then do a weighted average for all losses to achieve the final loss.
+            (To be added: a corresponding tutorial link)
+
         deep_supr_num: number of feature maps that will output during deep supervision head. The
-            value should be less than the number of up sample layers. Defaults to 1.
+            value should be larger than 0 and less than the number of up sample layers.
+            Defaults to 1.
         res_block: whether to use residual connection based convolution blocks during the network.
             Defaults to ``True``.
     """
@@ -95,6 +110,7 @@ def __init__(
         strides: Sequence[Union[Sequence[int], int]],
         upsample_kernel_size: Sequence[Union[Sequence[int], int]],
         norm_name: str = "instance",
+        deep_supervision: bool = False,
         deep_supr_num: int = 1,
         res_block: bool = False,
     ):
@@ -113,6 +129,7 @@ def __init__(
         self.bottleneck = self.get_bottleneck()
         self.upsamples = self.get_upsamples()
         self.output_block = self.get_output_block(0)
+        self.deep_supervision = deep_supervision
         self.deep_supervision_heads = self.get_deep_supervision_heads()
         self.deep_supr_num = deep_supr_num
         self.apply(self.initialize_weights)
@@ -140,6 +157,8 @@ def create_skips(index, downsamples, upsamples, superheads, bottleneck):
                 return bottleneck
             if index == 0:  # don't associate a supervision head with self.input_block
                 current_head, rest_heads = nn.Identity(), superheads
+            elif not self.deep_supervision:  # bypass supervision heads by passing nn.Identity in place of a real one
+                current_head, rest_heads = nn.Identity(), superheads[1:]
             else:
                 current_head, rest_heads = superheads[0], superheads[1:]
 
@@ -176,19 +195,21 @@ def check_kernel_stride(self):
     def check_deep_supr_num(self):
         deep_supr_num, strides = self.deep_supr_num, self.strides
         num_up_layers = len(strides) - 1
-        if deep_supr_num < 1 or deep_supr_num >= num_up_layers:
+        if deep_supr_num >= num_up_layers:
             raise AssertionError("deep_supr_num should be less than the number of up sample layers.")
+        if deep_supr_num < 1:
+            raise AssertionError("deep_supr_num should be larger than 0.")
 
     def forward(self, x):
         out = self.skip_layers(x)
-        return self.output_block(out)
-
-    def get_feature_maps(self):
-        """
-        Return the feature maps.
-
-        """
-        return self.heads[1 : self.deep_supr_num + 1]
+        out = self.output_block(out)
+        if self.training and self.deep_supervision:
+            out_all = [out]
+            feature_maps = self.heads[1 : self.deep_supr_num + 1]
+            for feature_map in feature_maps:
+                out_all.append(interpolate(feature_map, out.shape[2:]))
+            return torch.stack(out_all, dim=1)
+        return out
 
     def get_input_block(self):
         return self.conv_block(
diff --git a/monai/networks/nets/localnet.py b/monai/networks/nets/localnet.py
index ea8abca185..e9df68104d 100644
--- a/monai/networks/nets/localnet.py
+++ b/monai/networks/nets/localnet.py
@@ -99,7 +99,7 @@ def forward(self, x) -> torch.Tensor:
             if size % (2 ** self.extract_max_level) != 0:
                 raise ValueError(
                     f"given extract_max_level {self.extract_max_level}, "
-                    f"all input spatial dimension must be devidable by {2 ** self.extract_max_level}, "
+                    f"all input spatial dimension must be divisible by {2 ** self.extract_max_level}, "
                     f"got input of size {image_size}"
                 )
         mid_features = []  # 0 -> self.extract_max_level - 1
diff --git a/monai/networks/nets/regressor.py b/monai/networks/nets/regressor.py
index a1abadb6ba..d64ad2fc10 100644
--- a/monai/networks/nets/regressor.py
+++ b/monai/networks/nets/regressor.py
@@ -78,7 +78,7 @@ def __init__(
 
         padding = same_padding(kernel_size)
 
-        self.final_size = np.asarray(self.in_shape, np.int)
+        self.final_size = np.asarray(self.in_shape, dtype=int)
         self.reshape = Reshape(*self.out_shape)
 
         # encode stage
@@ -86,7 +86,7 @@ def __init__(
             layer = self._get_layer(echannel, c, s, i == len(channels) - 1)
             echannel = c  # use the output channel number as the input for the next loop
             self.net.add_module("layer_%i" % i, layer)
-            self.final_size = calculate_out_shape(self.final_size, kernel_size, s, padding)
+            self.final_size = calculate_out_shape(self.final_size, kernel_size, s, padding)  # type: ignore
 
         self.final = self._get_final_layer((echannel,) + self.final_size)
 
diff --git a/monai/networks/nets/segresnet.py b/monai/networks/nets/segresnet.py
index c7a085b569..4626a38abd 100644
--- a/monai/networks/nets/segresnet.py
+++ b/monai/networks/nets/segresnet.py
@@ -40,11 +40,11 @@ class SegResNet(nn.Module):
         use_conv_final: if add a final convolution block to output. Defaults to ``True``.
         blocks_down: number of down sample blocks in each layer. Defaults to ``[1,2,2,4]``.
         blocks_up: number of up sample blocks in each layer. Defaults to ``[1,1,1]``.
-        upsample_mode: [``"transpose"``, ``"nontrainable"``, ``"pixelshuffle"``]
+        upsample_mode: [``"deconv"``, ``"nontrainable"``, ``"pixelshuffle"``]
             The mode of upsampling manipulations.
             Using the ``nontrainable`` modes cannot guarantee the model's reproducibility. Defaults to``nontrainable``.
 
-            - ``transpose``, uses transposed convolution layers.
+            - ``deconv``, uses transposed convolution layers.
             - ``nontrainable``, uses non-trainable `linear` interpolation.
             - ``pixelshuffle``, uses :py:class:`monai.networks.blocks.SubpixelUpsample`.
 
@@ -192,11 +192,11 @@ class SegResNetVAE(SegResNet):
         use_conv_final: if add a final convolution block to output. Defaults to ``True``.
         blocks_down: number of down sample blocks in each layer. Defaults to ``[1,2,2,4]``.
         blocks_up: number of up sample blocks in each layer. Defaults to ``[1,1,1]``.
-        upsample_mode: [``"transpose"``, ``"nontrainable"``, ``"pixelshuffle"``]
+        upsample_mode: [``"deconv"``, ``"nontrainable"``, ``"pixelshuffle"``]
             The mode of upsampling manipulations.
             Using the ``nontrainable`` modes cannot guarantee the model's reproducibility. Defaults to `nontrainable`.
 
-            - ``transpose``, uses transposed convolution layers.
+            - ``deconv``, uses transposed convolution layers.
             - ``nontrainable``, uses non-trainable `linear` interpolation.
             - ``pixelshuffle``, uses :py:class:`monai.networks.blocks.SubpixelUpsample`.
 
diff --git a/monai/networks/nets/varautoencoder.py b/monai/networks/nets/varautoencoder.py
index b68350e8b1..30ee806dbb 100644
--- a/monai/networks/nets/varautoencoder.py
+++ b/monai/networks/nets/varautoencoder.py
@@ -46,7 +46,7 @@ def __init__(
         self.in_channels, *self.in_shape = in_shape
 
         self.latent_size = latent_size
-        self.final_size = np.asarray(self.in_shape, np.int)
+        self.final_size = np.asarray(self.in_shape, dtype=int)
 
         super().__init__(
             dimensions,
@@ -68,7 +68,7 @@ def __init__(
         padding = same_padding(self.kernel_size)
 
         for s in strides:
-            self.final_size = calculate_out_shape(self.final_size, self.kernel_size, s, padding)
+            self.final_size = calculate_out_shape(self.final_size, self.kernel_size, s, padding)  # type: ignore
 
         linear_size = int(np.product(self.final_size)) * self.encoded_channels
         self.mu = nn.Linear(linear_size, self.latent_size)
diff --git a/monai/networks/utils.py b/monai/networks/utils.py
index 175d3d8b73..847bfc97c2 100644
--- a/monai/networks/utils.py
+++ b/monai/networks/utils.py
@@ -150,7 +150,7 @@ def to_norm_affine(
         ValueError: When ``src_size`` or ``dst_size`` dimensions differ from ``affine``.
 
     """
-    if not torch.is_tensor(affine):
+    if not isinstance(affine, torch.Tensor):
         raise TypeError(f"affine must be a torch.Tensor but is {type(affine).__name__}.")
     if affine.ndimension() != 3 or affine.shape[1] != affine.shape[2]:
         raise ValueError(f"affine must be Nxdxd, got {tuple(affine.shape)}.")
diff --git a/monai/optimizers/lr_finder.py b/monai/optimizers/lr_finder.py
index 6ad4132dd0..9e753a1ced 100644
--- a/monai/optimizers/lr_finder.py
+++ b/monai/optimizers/lr_finder.py
@@ -5,7 +5,6 @@
 import numpy as np
 import torch
 import torch.nn as nn
-from numpy.core.arrayprint import _none_or_positive_arg
 from torch.optim import Optimizer
 from torch.utils.data import DataLoader
 
@@ -363,7 +362,7 @@ def _set_learning_rate(self, new_lrs: Union[float, list]) -> None:
         for param_group, new_lr in zip(self.optimizer.param_groups, new_lrs):
             param_group["lr"] = new_lr
 
-    def _check_for_scheduler(self) -> _none_or_positive_arg:
+    def _check_for_scheduler(self):
         """Check optimizer doesn't already have scheduler."""
         for param_group in self.optimizer.param_groups:
             if "initial_lr" in param_group:
diff --git a/monai/transforms/__init__.py b/monai/transforms/__init__.py
index 9eaedd6b15..6f7c2a4f61 100644
--- a/monai/transforms/__init__.py
+++ b/monai/transforms/__init__.py
@@ -138,8 +138,8 @@
     ThresholdIntensityD,
     ThresholdIntensityDict,
 )
-from .io.array import LoadImage
-from .io.dictionary import LoadImaged, LoadImageD, LoadImageDict
+from .io.array import LoadImage, SaveImage
+from .io.dictionary import LoadImaged, LoadImageD, LoadImageDict, SaveImaged, SaveImageD, SaveImageDict
 from .post.array import (
     Activations,
     AsDiscrete,
@@ -298,6 +298,9 @@
     Lambdad,
     LambdaD,
     LambdaDict,
+    RandLambdad,
+    RandLambdaD,
+    RandLambdaDict,
     RepeatChanneld,
     RepeatChannelD,
     RepeatChannelDict,
@@ -335,6 +338,7 @@
     in_bounds,
     is_empty,
     map_binary_to_indices,
+    map_spatial_axes,
     rand_choice,
     rescale_array,
     rescale_array_int_max,
diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py
index 3e23377b36..2d1fe4eccd 100644
--- a/monai/transforms/compose.py
+++ b/monai/transforms/compose.py
@@ -231,6 +231,26 @@ def randomize(self, data: Optional[Any] = None) -> None:
                     f'Transform "{tfm_name}" in Compose not randomized\n{tfm_name}.{type_error}.', RuntimeWarning
                 )
 
+    def flatten(self):
+        """Return a Composition with a simple list of transforms, as opposed to any nested Compositions.
+
+        e.g., `t1 = Compose([x, x, x, x, Compose([Compose([x, x]), x, x])]).flatten()`
+        will result in the equivalent of `t1 = Compose([x, x, x, x, x, x, x, x])`.
+
+        """
+        new_transforms = []
+        for t in self.transforms:
+            if isinstance(t, Compose):
+                new_transforms += t.flatten().transforms
+            else:
+                new_transforms.append(t)
+
+        return Compose(new_transforms)
+
+    def __len__(self):
+        """Return number of transformations."""
+        return len(self.flatten().transforms)
+
     def __call__(self, input_):
         for _transform in self.transforms:
             input_ = apply_transform(_transform, input_)
diff --git a/monai/transforms/croppad/array.py b/monai/transforms/croppad/array.py
index e59eb89ac7..b4444803a4 100644
--- a/monai/transforms/croppad/array.py
+++ b/monai/transforms/croppad/array.py
@@ -16,6 +16,7 @@
 from typing import Any, Callable, List, Optional, Sequence, Tuple, Union
 
 import numpy as np
+import torch
 
 from monai.config import IndexSelection
 from monai.data.utils import get_random_patch, get_valid_patch_size
@@ -128,7 +129,7 @@ def __init__(
         self.spatial_border = spatial_border
         self.mode: NumpyPadMode = NumpyPadMode(mode)
 
-    def __call__(self, img: np.ndarray, mode: Optional[Union[NumpyPadMode, str]] = None) -> np.ndarray:
+    def __call__(self, img: np.ndarray, mode: Optional[Union[NumpyPadMode, str]] = None):
         """
         Args:
             img: data to be transformed, assuming `img` is channel-first and
@@ -219,10 +220,10 @@ class SpatialCrop(Transform):
 
     def __init__(
         self,
-        roi_center: Optional[Sequence[int]] = None,
-        roi_size: Optional[Sequence[int]] = None,
-        roi_start: Optional[Sequence[int]] = None,
-        roi_end: Optional[Sequence[int]] = None,
+        roi_center: Union[Sequence[int], np.ndarray, None] = None,
+        roi_size: Union[Sequence[int], np.ndarray, None] = None,
+        roi_start: Union[Sequence[int], np.ndarray, None] = None,
+        roi_end: Union[Sequence[int], np.ndarray, None] = None,
     ) -> None:
         """
         Args:
@@ -242,14 +243,14 @@ def __init__(
             self.roi_start = np.maximum(np.asarray(roi_start, dtype=np.int16), 0)
             self.roi_end = np.maximum(np.asarray(roi_end, dtype=np.int16), self.roi_start)
 
-    def __call__(self, img: np.ndarray) -> np.ndarray:
+    def __call__(self, img: Union[np.ndarray, torch.Tensor]) -> np.ndarray:
         """
         Apply the transform to `img`, assuming `img` is channel-first and
         slicing doesn't apply to the channel dim.
         """
         sd = min(len(self.roi_start), len(self.roi_end), len(img.shape[1:]))  # spatial dims
         slices = [slice(None)] + [slice(s, e) for s, e in zip(self.roi_start[:sd], self.roi_end[:sd])]
-        return img[tuple(slices)]
+        return np.asarray(img[tuple(slices)])
 
 
 class CenterSpatialCrop(Transform):
@@ -264,7 +265,7 @@ class CenterSpatialCrop(Transform):
     def __init__(self, roi_size: Union[Sequence[int], int]) -> None:
         self.roi_size = roi_size
 
-    def __call__(self, img: np.ndarray) -> np.ndarray:
+    def __call__(self, img: np.ndarray):
         """
         Apply the transform to `img`, assuming `img` is channel-first and
         slicing doesn't apply to the channel dim.
@@ -306,7 +307,7 @@ def randomize(self, img_size: Sequence[int]) -> None:
             valid_size = get_valid_patch_size(img_size, self._size)
             self._slices = (slice(None),) + get_random_patch(img_size, valid_size, self.R)
 
-    def __call__(self, img: np.ndarray) -> np.ndarray:
+    def __call__(self, img: np.ndarray):
         """
         Apply the transform to `img`, assuming `img` is channel-first and
         slicing doesn't apply to the channel dim.
@@ -590,6 +591,8 @@ def __call__(
         """
         if label is None:
             label = self.label
+        if label is None:
+            raise ValueError("label should be provided.")
         if image is None:
             image = self.image
         if fg_indices is None or bg_indices is None:
@@ -602,7 +605,7 @@ def __call__(
         results: List[np.ndarray] = []
         if self.centers is not None:
             for center in self.centers:
-                cropper = SpatialCrop(roi_center=tuple(center), roi_size=self.spatial_size)
+                cropper = SpatialCrop(roi_center=tuple(center), roi_size=self.spatial_size)  # type: ignore
                 results.append(cropper(img))
 
         return results
diff --git a/monai/transforms/croppad/dictionary.py b/monai/transforms/croppad/dictionary.py
index 8bf33dd632..1faed25605 100644
--- a/monai/transforms/croppad/dictionary.py
+++ b/monai/transforms/croppad/dictionary.py
@@ -412,8 +412,8 @@ def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.nda
         box_start, box_end = generate_spatial_bounding_box(
             d[self.source_key], self.select_fn, self.channel_indices, self.margin
         )
-        d[self.start_coord_key] = box_start
-        d[self.end_coord_key] = box_end
+        d[self.start_coord_key] = np.asarray(box_start)
+        d[self.end_coord_key] = np.asarray(box_end)
         cropper = SpatialCrop(roi_start=box_start, roi_end=box_end)
         for key in self.keys:
             d[key] = cropper(d[key])
@@ -583,7 +583,7 @@ def __call__(self, data: Mapping[Hashable, np.ndarray]) -> List[Dict[Hashable, n
             if key in self.keys:
                 img = d[key]
                 for i, center in enumerate(self.centers):
-                    cropper = SpatialCrop(roi_center=tuple(center), roi_size=self.spatial_size)
+                    cropper = SpatialCrop(roi_center=tuple(center), roi_size=self.spatial_size)  # type: ignore
                     results[i][key] = cropper(img)
             else:
                 for i in range(self.num_samples):
diff --git a/monai/transforms/intensity/array.py b/monai/transforms/intensity/array.py
index 205b719246..87091f6237 100644
--- a/monai/transforms/intensity/array.py
+++ b/monai/transforms/intensity/array.py
@@ -20,6 +20,7 @@
 import numpy as np
 import torch
 
+from monai.config import DtypeLike
 from monai.networks.layers import GaussianFilter, HilbertTransform, SavitzkyGolayFilter
 from monai.transforms.compose import Randomizable, Transform
 from monai.transforms.utils import rescale_array
@@ -97,7 +98,7 @@ def __call__(self, img: np.ndarray) -> np.ndarray:
         """
         Apply the transform to `img`.
         """
-        return (img + self.offset).astype(img.dtype)
+        return np.asarray((img + self.offset), dtype=img.dtype)
 
 
 class RandShiftIntensity(Randomizable, Transform):
@@ -165,9 +166,9 @@ def __call__(self, img: np.ndarray) -> np.ndarray:
 
         """
         if self.minv is not None and self.maxv is not None:
-            return rescale_array(img, self.minv, self.maxv, img.dtype)
+            return np.asarray(rescale_array(img, self.minv, self.maxv, img.dtype))
         if self.factor is not None:
-            return (img * (1 + self.factor)).astype(img.dtype)
+            return np.asarray(img * (1 + self.factor), dtype=img.dtype)
         raise ValueError("Incompatible values: minv=None or maxv=None and factor=None.")
 
 
@@ -229,11 +230,11 @@ class NormalizeIntensity(Transform):
 
     def __init__(
         self,
-        subtrahend: Optional[Sequence] = None,
-        divisor: Optional[Sequence] = None,
+        subtrahend: Union[Sequence, np.ndarray, None] = None,
+        divisor: Union[Sequence, np.ndarray, None] = None,
         nonzero: bool = False,
         channel_wise: bool = False,
-        dtype: np.dtype = np.float32,
+        dtype: DtypeLike = np.float32,
     ) -> None:
         self.subtrahend = subtrahend
         self.divisor = divisor
@@ -304,7 +305,9 @@ def __call__(self, img: np.ndarray) -> np.ndarray:
         """
         Apply the transform to `img`.
         """
-        return np.where(img > self.threshold if self.above else img < self.threshold, img, self.cval).astype(img.dtype)
+        return np.asarray(
+            np.where(img > self.threshold if self.above else img < self.threshold, img, self.cval), dtype=img.dtype
+        )
 
 
 class ScaleIntensityRange(Transform):
@@ -327,7 +330,7 @@ def __init__(self, a_min: float, a_max: float, b_min: float, b_max: float, clip:
         self.b_max = b_max
         self.clip = clip
 
-    def __call__(self, img: np.ndarray) -> np.ndarray:
+    def __call__(self, img: np.ndarray):
         """
         Apply the transform to `img`.
         """
@@ -338,8 +341,7 @@ def __call__(self, img: np.ndarray) -> np.ndarray:
         img = (img - self.a_min) / (self.a_max - self.a_min)
         img = img * (self.b_max - self.b_min) + self.b_min
         if self.clip:
-            img = np.clip(img, self.b_min, self.b_max)
-
+            img = np.asarray(np.clip(img, self.b_min, self.b_max))
         return img
 
 
@@ -358,7 +360,7 @@ def __init__(self, gamma: float) -> None:
             raise AssertionError("gamma must be a float or int number.")
         self.gamma = gamma
 
-    def __call__(self, img: np.ndarray) -> np.ndarray:
+    def __call__(self, img: np.ndarray):
         """
         Apply the transform to `img`.
         """
@@ -483,7 +485,7 @@ def __init__(
         self.clip = clip
         self.relative = relative
 
-    def __call__(self, img: np.ndarray) -> np.ndarray:
+    def __call__(self, img: np.ndarray):
         """
         Apply the transform to `img`.
         """
@@ -500,7 +502,7 @@ def __call__(self, img: np.ndarray) -> np.ndarray:
         img = scalar(img)
 
         if self.clip:
-            img = np.clip(img, self.b_min, self.b_max)
+            img = np.asarray(np.clip(img, self.b_min, self.b_max))
 
         return img
 
@@ -513,36 +515,44 @@ class MaskIntensity(Transform):
     data will be set to `0`, others will keep the original value.
 
     Args:
-        mask_data: if mask data is single channel, apply to evey channel
-            of input image. if multiple channels, the channel number must
-            match input data. mask_data will be converted to `bool` values
+        mask_data: if `mask_data` is single channel, apply to every channel
+            of input image. if multiple channels, the number of channels must
+            match the input data. `mask_data` will be converted to `bool` values
             by `mask_data > 0` before applying transform to input image.
 
     """
 
-    def __init__(self, mask_data: np.ndarray) -> None:
+    def __init__(self, mask_data: Optional[np.ndarray]) -> None:
         self.mask_data = mask_data
 
     def __call__(self, img: np.ndarray, mask_data: Optional[np.ndarray] = None) -> np.ndarray:
         """
         Args:
-            mask_data: if mask data is single channel, apply to evey channel
+            mask_data: if mask data is single channel, apply to every channel
                 of input image. if multiple channels, the channel number must
                 match input data. mask_data will be converted to `bool` values
                 by `mask_data > 0` before applying transform to input image.
 
         Raises:
-            ValueError: When ``mask_data`` and ``img`` channels differ and ``mask_data`` is not single channel.
-
-        """
-        mask_data_ = self.mask_data > 0 if mask_data is None else mask_data > 0
+            - ValueError: When both ``mask_data`` and ``self.mask_data`` are None.
+            - ValueError: When ``mask_data`` and ``img`` channels differ and ``mask_data`` is not single channel.
+
+        """
+        if self.mask_data is None and mask_data is None:
+            raise ValueError("Unknown mask_data.")
+        mask_data_ = np.array([[1]])
+        if self.mask_data is not None and mask_data is None:
+            mask_data_ = self.mask_data > 0
+        if mask_data is not None:
+            mask_data_ = mask_data > 0
+        mask_data_ = np.asarray(mask_data_)
         if mask_data_.shape[0] != 1 and mask_data_.shape[0] != img.shape[0]:
             raise ValueError(
                 "When mask_data is not single channel, mask_data channels must match img, "
                 f"got img={img.shape[0]} mask_data={mask_data_.shape[0]}."
             )
 
-        return img * mask_data_
+        return np.asarray(img * mask_data_)
 
 
 class SavitzkyGolaySmooth(Transform):
@@ -567,7 +577,7 @@ def __init__(self, window_length: int, order: int, axis: int = 1, mode: str = "z
         self.axis = axis
         self.mode = mode
 
-    def __call__(self, img: np.ndarray) -> np.ndarray:
+    def __call__(self, img: np.ndarray):
         """
         Args:
             img: numpy.ndarray containing input data. Must be real and in shape [channels, spatial1, spatial2, ...].
@@ -606,7 +616,7 @@ def __init__(self, axis: int = 1, n: Union[int, None] = None) -> None:
         self.axis = axis
         self.n = n
 
-    def __call__(self, img: np.ndarray) -> np.ndarray:
+    def __call__(self, img: np.ndarray):
         """
 
         Args:
@@ -641,7 +651,7 @@ def __init__(self, sigma: Union[Sequence[float], float] = 1.0, approx: str = "er
         self.sigma = sigma
         self.approx = approx
 
-    def __call__(self, img: np.ndarray) -> np.ndarray:
+    def __call__(self, img: np.ndarray):
         gaussian_filter = GaussianFilter(img.ndim - 1, self.sigma, approx=self.approx)
         input_data = torch.as_tensor(np.ascontiguousarray(img), dtype=torch.float).unsqueeze(0)
         return gaussian_filter(input_data).squeeze(0).detach().numpy()
@@ -682,7 +692,7 @@ def randomize(self, data: Optional[Any] = None) -> None:
         self.y = self.R.uniform(low=self.sigma_y[0], high=self.sigma_y[1])
         self.z = self.R.uniform(low=self.sigma_z[0], high=self.sigma_z[1])
 
-    def __call__(self, img: np.ndarray) -> np.ndarray:
+    def __call__(self, img: np.ndarray):
         self.randomize()
         if not self._do_transform:
             return img
@@ -729,7 +739,7 @@ def __init__(
         self.alpha = alpha
         self.approx = approx
 
-    def __call__(self, img: np.ndarray) -> np.ndarray:
+    def __call__(self, img: np.ndarray):
         gaussian_filter1 = GaussianFilter(img.ndim - 1, self.sigma1, approx=self.approx)
         gaussian_filter2 = GaussianFilter(img.ndim - 1, self.sigma2, approx=self.approx)
         input_data = torch.as_tensor(np.ascontiguousarray(img), dtype=torch.float).unsqueeze(0)
@@ -796,7 +806,7 @@ def randomize(self, data: Optional[Any] = None) -> None:
         self.z2 = self.R.uniform(low=sigma2_z[0], high=sigma2_z[1])
         self.a = self.R.uniform(low=self.alpha[0], high=self.alpha[1])
 
-    def __call__(self, img: np.ndarray) -> np.ndarray:
+    def __call__(self, img: np.ndarray):
         self.randomize()
         if not self._do_transform:
             return img
@@ -848,4 +858,6 @@ def __call__(self, img: np.ndarray) -> np.ndarray:
         img_min, img_max = img.min(), img.max()
         reference_control_points_scaled = self.reference_control_points * (img_max - img_min) + img_min
         floating_control_points_scaled = self.floating_control_points * (img_max - img_min) + img_min
-        return np.interp(img, reference_control_points_scaled, floating_control_points_scaled).astype(img.dtype)
+        return np.asarray(
+            np.interp(img, reference_control_points_scaled, floating_control_points_scaled), dtype=img.dtype
+        )
diff --git a/monai/transforms/intensity/dictionary.py b/monai/transforms/intensity/dictionary.py
index 18e2250084..1c9b31c120 100644
--- a/monai/transforms/intensity/dictionary.py
+++ b/monai/transforms/intensity/dictionary.py
@@ -16,12 +16,12 @@
 """
 
 from collections.abc import Iterable
-from typing import Any, Dict, Hashable, Mapping, Optional, Sequence, Tuple, Union
+from typing import Any, Dict, Hashable, List, Mapping, Optional, Sequence, Tuple, Union
 
 import numpy as np
 import torch
 
-from monai.config import KeysCollection
+from monai.config import DtypeLike, KeysCollection
 from monai.transforms.compose import MapTransform, Randomizable
 from monai.transforms.intensity.array import (
     AdjustContrast,
@@ -35,7 +35,7 @@
     ShiftIntensity,
     ThresholdIntensity,
 )
-from monai.utils import dtype_torch_to_numpy, ensure_tuple_size
+from monai.utils import dtype_torch_to_numpy, ensure_tuple_rep, ensure_tuple_size
 
 __all__ = [
     "RandGaussianNoised",
@@ -110,27 +110,29 @@ def __init__(
     ) -> None:
         super().__init__(keys)
         self.prob = prob
-        self.mean = ensure_tuple_size(mean, len(self.keys))
+        self.mean = ensure_tuple_rep(mean, len(self.keys))
         self.std = std
         self._do_transform = False
-        self._noise: Optional[np.ndarray] = None
+        self._noise: List[np.ndarray] = []
 
     def randomize(self, im_shape: Sequence[int]) -> None:
         self._do_transform = self.R.random() < self.prob
-        self._noise = self.R.normal(self.mean, self.R.uniform(0, self.std), size=im_shape)
+        self._noise.clear()
+        for m in self.mean:
+            self._noise.append(self.R.normal(m, self.R.uniform(0, self.std), size=im_shape))
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
 
         image_shape = d[self.keys[0]].shape  # image shape from the first data key
         self.randomize(image_shape)
-        if self._noise is None:
+        if len(self._noise) != len(self.keys):
             raise AssertionError
         if not self._do_transform:
             return d
-        for key in self.keys:
+        for noise, key in zip(self._noise, self.keys):
             dtype = dtype_torch_to_numpy(d[key].dtype) if isinstance(d[key], torch.Tensor) else d[key].dtype
-            d[key] = d[key] + self._noise.astype(dtype)
+            d[key] = d[key] + noise.astype(dtype)
         return d
 
 
@@ -294,7 +296,7 @@ def __init__(
         divisor: Optional[np.ndarray] = None,
         nonzero: bool = False,
         channel_wise: bool = False,
-        dtype: np.dtype = np.float32,
+        dtype: DtypeLike = np.float32,
     ) -> None:
         super().__init__(keys)
         self.normalizer = NormalizeIntensity(subtrahend, divisor, nonzero, channel_wise, dtype)
@@ -474,7 +476,7 @@ class MaskIntensityd(MapTransform):
     Args:
         keys: keys of the corresponding items to be transformed.
             See also: :py:class:`monai.transforms.compose.MapTransform`
-        mask_data: if mask data is single channel, apply to evey channel
+        mask_data: if mask data is single channel, apply to every channel
             of input image. if multiple channels, the channel number must
             match input data. mask_data will be converted to `bool` values
             by `mask_data > 0` before applying transform to input image.
diff --git a/monai/transforms/io/array.py b/monai/transforms/io/array.py
index 3b359cc460..9c14f7a689 100644
--- a/monai/transforms/io/array.py
+++ b/monai/transforms/io/array.py
@@ -13,18 +13,24 @@
 https://github.com/Project-MONAI/MONAI/wiki/MONAI_Design
 """
 
-from typing import List, Optional, Sequence, Union
+from typing import Dict, List, Optional, Sequence, Union
 
 import numpy as np
+import torch
 
+from monai.config import DtypeLike
 from monai.data.image_reader import ImageReader, ITKReader, NibabelReader, NumpyReader, PILReader
+from monai.data.nifti_saver import NiftiSaver
+from monai.data.png_saver import PNGSaver
 from monai.transforms.compose import Transform
-from monai.utils import ensure_tuple, optional_import
+from monai.utils import GridSampleMode, GridSamplePadMode
+from monai.utils import ImageMetaKey as Key
+from monai.utils import InterpolateMode, ensure_tuple, optional_import
 
 nib, _ = optional_import("nibabel")
 Image, _ = optional_import("PIL.Image")
 
-__all__ = ["LoadImage"]
+__all__ = ["LoadImage", "SaveImage"]
 
 
 class LoadImage(Transform):
@@ -42,7 +48,7 @@ def __init__(
         self,
         reader: Optional[Union[ImageReader, str]] = None,
         image_only: bool = False,
-        dtype: np.dtype = np.float32,
+        dtype: DtypeLike = np.float32,
         *args,
         **kwargs,
     ) -> None:
@@ -125,5 +131,95 @@ def __call__(
 
         if self.image_only:
             return img_array
-        meta_data["filename_or_obj"] = ensure_tuple(filename)[0]
+        meta_data[Key.FILENAME_OR_OBJ] = ensure_tuple(filename)[0]
         return img_array, meta_data
+
+
+class SaveImage(Transform):
+    """
+    Save transformed data into files, support NIfTI and PNG formats.
+    It can work for both numpy array and PyTorch Tensor in both pre-transform chain
+    and post transform chain.
+
+    Args:
+        output_dir: output image directory.
+        output_postfix: a string appended to all output file names, default to `trans`.
+        output_ext: output file extension name, available extensions: `.nii.gz`, `.nii`, `.png`.
+        resample: whether to resample before saving the data array.
+            if saving PNG format image, based on the `spatial_shape` from metadata.
+            if saving NIfTI format image, based on the `original_affine` from metadata.
+        mode: This option is used when ``resample = True``. Defaults to ``"nearest"``.
+
+            - NIfTI files {``"bilinear"``, ``"nearest"``}
+                Interpolation mode to calculate output values.
+                See also: https://pytorch.org/docs/stable/nn.functional.html#grid-sample
+            - PNG files {``"nearest"``, ``"linear"``, ``"bilinear"``, ``"bicubic"``, ``"trilinear"``, ``"area"``}
+                The interpolation mode.
+                See also: https://pytorch.org/docs/stable/nn.functional.html#interpolate
+
+        padding_mode: This option is used when ``resample = True``. Defaults to ``"border"``.
+
+            - NIfTI files {``"zeros"``, ``"border"``, ``"reflection"``}
+                Padding mode for outside grid values.
+                See also: https://pytorch.org/docs/stable/nn.functional.html#grid-sample
+            - PNG files
+                This option is ignored.
+
+        scale: {``255``, ``65535``} postprocess data by clipping to [0, 1] and scaling
+            [0, 255] (uint8) or [0, 65535] (uint16). Default is None to disable scaling.
+            it's used for PNG format only.
+        dtype: data type during resampling computation. Defaults to ``np.float64`` for best precision.
+            if None, use the data type of input data. To be compatible with other modules,
+            the output data type is always ``np.float32``.
+            it's used for NIfTI format only.
+        output_dtype: data type for saving data. Defaults to ``np.float32``.
+            it's used for NIfTI format only.
+        save_batch: whether the import image is a batch data, default to `False`.
+            usually pre-transforms run for channel first data, while post-transforms run for batch data.
+
+    """
+
+    def __init__(
+        self,
+        output_dir: str = "./",
+        output_postfix: str = "trans",
+        output_ext: str = ".nii.gz",
+        resample: bool = True,
+        mode: Union[GridSampleMode, InterpolateMode, str] = "nearest",
+        padding_mode: Union[GridSamplePadMode, str] = GridSamplePadMode.BORDER,
+        scale: Optional[int] = None,
+        dtype: DtypeLike = np.float64,
+        output_dtype: DtypeLike = np.float32,
+        save_batch: bool = False,
+    ) -> None:
+        self.saver: Union[NiftiSaver, PNGSaver]
+        if output_ext in (".nii.gz", ".nii"):
+            self.saver = NiftiSaver(
+                output_dir=output_dir,
+                output_postfix=output_postfix,
+                output_ext=output_ext,
+                resample=resample,
+                mode=GridSampleMode(mode),
+                padding_mode=padding_mode,
+                dtype=dtype,
+                output_dtype=output_dtype,
+            )
+        elif output_ext == ".png":
+            self.saver = PNGSaver(
+                output_dir=output_dir,
+                output_postfix=output_postfix,
+                output_ext=output_ext,
+                resample=resample,
+                mode=InterpolateMode(mode),
+                scale=scale,
+            )
+        else:
+            raise ValueError(f"unsupported output extension: {output_ext}.")
+
+        self.save_batch = save_batch
+
+    def __call__(self, img: Union[torch.Tensor, np.ndarray], meta_data: Optional[Dict] = None):
+        if self.save_batch:
+            self.saver.save_batch(img, meta_data)
+        else:
+            self.saver.save(img, meta_data)
diff --git a/monai/transforms/io/dictionary.py b/monai/transforms/io/dictionary.py
index 62ac4c8562..d3220aa682 100644
--- a/monai/transforms/io/dictionary.py
+++ b/monai/transforms/io/dictionary.py
@@ -19,15 +19,19 @@
 
 import numpy as np
 
-from monai.config import KeysCollection
+from monai.config import DtypeLike, KeysCollection
 from monai.data.image_reader import ImageReader
 from monai.transforms.compose import MapTransform
-from monai.transforms.io.array import LoadImage
+from monai.transforms.io.array import LoadImage, SaveImage
+from monai.utils import GridSampleMode, GridSamplePadMode, InterpolateMode
 
 __all__ = [
     "LoadImaged",
     "LoadImageD",
     "LoadImageDict",
+    "SaveImaged",
+    "SaveImageD",
+    "SaveImageDict",
 ]
 
 
@@ -52,7 +56,7 @@ def __init__(
         self,
         keys: KeysCollection,
         reader: Optional[Union[ImageReader, str]] = None,
-        dtype: Optional[np.dtype] = np.float32,
+        dtype: DtypeLike = np.float32,
         meta_key_postfix: str = "meta_dict",
         overwriting: bool = False,
         *args,
@@ -106,4 +110,91 @@ def __call__(self, data, reader: Optional[ImageReader] = None):
         return d
 
 
+class SaveImaged(MapTransform):
+    """
+    Dictionary-based wrapper of :py:class:`monai.transforms.SaveImage`.
+
+    Args:
+        keys: keys of the corresponding items to be transformed.
+            See also: :py:class:`monai.transforms.compose.MapTransform`
+        meta_key_postfix: `key_{postfix}` was used to store the metadata in `LoadImaged`.
+            So need the key to extract metadata to save images, default is `meta_dict`.
+            The meta data is a dictionary object, if no corresponding metadata, set to `None`.
+            For example, for data with key `image`, the metadata by default is in `image_meta_dict`.
+        output_dir: output image directory.
+        output_postfix: a string appended to all output file names, default to `trans`.
+        output_ext: output file extension name, available extensions: `.nii.gz`, `.nii`, `.png`.
+        resample: whether to resample before saving the data array.
+            if saving PNG format image, based on the `spatial_shape` from metadata.
+            if saving NIfTI format image, based on the `original_affine` from metadata.
+        mode: This option is used when ``resample = True``. Defaults to ``"nearest"``.
+
+            - NIfTI files {``"bilinear"``, ``"nearest"``}
+                Interpolation mode to calculate output values.
+                See also: https://pytorch.org/docs/stable/nn.functional.html#grid-sample
+            - PNG files {``"nearest"``, ``"linear"``, ``"bilinear"``, ``"bicubic"``, ``"trilinear"``, ``"area"``}
+                The interpolation mode.
+                See also: https://pytorch.org/docs/stable/nn.functional.html#interpolate
+
+        padding_mode: This option is used when ``resample = True``. Defaults to ``"border"``.
+
+            - NIfTI files {``"zeros"``, ``"border"``, ``"reflection"``}
+                Padding mode for outside grid values.
+                See also: https://pytorch.org/docs/stable/nn.functional.html#grid-sample
+            - PNG files
+                This option is ignored.
+
+        scale: {``255``, ``65535``} postprocess data by clipping to [0, 1] and scaling
+            [0, 255] (uint8) or [0, 65535] (uint16). Default is None to disable scaling.
+            it's used for PNG format only.
+        dtype: data type during resampling computation. Defaults to ``np.float64`` for best precision.
+            if None, use the data type of input data. To be compatible with other modules,
+            the output data type is always ``np.float32``.
+            it's used for NIfTI format only.
+        output_dtype: data type for saving data. Defaults to ``np.float32``.
+            it's used for NIfTI format only.
+        save_batch: whether the import image is a batch data, default to `False`.
+            usually pre-transforms run for channel first data, while post-transforms run for batch data.
+
+    """
+
+    def __init__(
+        self,
+        keys: KeysCollection,
+        meta_key_postfix: str = "meta_dict",
+        output_dir: str = "./",
+        output_postfix: str = "trans",
+        output_ext: str = ".nii.gz",
+        resample: bool = True,
+        mode: Union[GridSampleMode, InterpolateMode, str] = "nearest",
+        padding_mode: Union[GridSamplePadMode, str] = GridSamplePadMode.BORDER,
+        scale: Optional[int] = None,
+        dtype: DtypeLike = np.float64,
+        output_dtype: DtypeLike = np.float32,
+        save_batch: bool = False,
+    ) -> None:
+        super().__init__(keys)
+        self.meta_key_postfix = meta_key_postfix
+        self._saver = SaveImage(
+            output_dir=output_dir,
+            output_postfix=output_postfix,
+            output_ext=output_ext,
+            resample=resample,
+            mode=mode,
+            padding_mode=padding_mode,
+            scale=scale,
+            dtype=dtype,
+            output_dtype=output_dtype,
+            save_batch=save_batch,
+        )
+
+    def __call__(self, data):
+        d = dict(data)
+        for key in self.keys:
+            meta_data = d[f"{key}_{self.meta_key_postfix}"] if self.meta_key_postfix is not None else None
+            self._saver(img=d[key], meta_data=meta_data)
+        return d
+
+
 LoadImageD = LoadImageDict = LoadImaged
+SaveImageD = SaveImageDict = SaveImaged
diff --git a/monai/transforms/spatial/array.py b/monai/transforms/spatial/array.py
index 3e1ded4e94..df10480188 100644
--- a/monai/transforms/spatial/array.py
+++ b/monai/transforms/spatial/array.py
@@ -19,7 +19,7 @@
 import numpy as np
 import torch
 
-from monai.config import USE_COMPILED
+from monai.config import USE_COMPILED, DtypeLike
 from monai.data.utils import compute_shape_offset, to_affine_nd, zoom_affine
 from monai.networks.layers import AffineTransform, GaussianFilter, grid_pull
 from monai.transforms.compose import Randomizable, Transform
@@ -31,6 +31,7 @@
     create_scale,
     create_shear,
     create_translate,
+    map_spatial_axes,
 )
 from monai.utils import (
     GridSampleMode,
@@ -81,7 +82,7 @@ def __init__(
         mode: Union[GridSampleMode, str] = GridSampleMode.BILINEAR,
         padding_mode: Union[GridSamplePadMode, str] = GridSamplePadMode.BORDER,
         align_corners: bool = False,
-        dtype: Optional[np.dtype] = np.float64,
+        dtype: DtypeLike = np.float64,
     ) -> None:
         """
         Args:
@@ -123,7 +124,7 @@ def __call__(
         mode: Optional[Union[GridSampleMode, str]] = None,
         padding_mode: Optional[Union[GridSamplePadMode, str]] = None,
         align_corners: Optional[bool] = None,
-        dtype: Optional[np.dtype] = None,
+        dtype: DtypeLike = None,
     ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
         """
         Args:
@@ -192,7 +193,7 @@ def __call__(
             torch.as_tensor(np.ascontiguousarray(transform).astype(_dtype)),
             spatial_size=output_shape,
         )
-        output_data = output_data.squeeze(0).detach().cpu().numpy().astype(np.float32)
+        output_data = np.asarray(output_data.squeeze(0).detach().cpu().numpy(), dtype=np.float32)  # type: ignore
         new_affine = to_affine_nd(affine, new_affine)
         return output_data, affine, new_affine
 
@@ -285,14 +286,19 @@ def __call__(
 class Flip(Transform):
     """
     Reverses the order of elements along the given spatial axis. Preserves shape.
-    Uses ``np.flip`` in practice. See numpy.flip for additional details.
-    https://docs.scipy.org/doc/numpy/reference/generated/numpy.flip.html
+    Uses ``np.flip`` in practice. See numpy.flip for additional details:
+    https://docs.scipy.org/doc/numpy/reference/generated/numpy.flip.html.
 
     Args:
         spatial_axis: spatial axes along which to flip over. Default is None.
+            The default `axis=None` will flip over all of the axes of the input array.
+            If axis is negative it counts from the last to the first axis.
+            If axis is a tuple of ints, flipping is performed on all of the axes
+            specified in the tuple.
+
     """
 
-    def __init__(self, spatial_axis: Optional[Union[Sequence[int], int]]) -> None:
+    def __init__(self, spatial_axis: Optional[Union[Sequence[int], int]] = None) -> None:
         self.spatial_axis = spatial_axis
 
     def __call__(self, img: np.ndarray) -> np.ndarray:
@@ -300,10 +306,9 @@ def __call__(self, img: np.ndarray) -> np.ndarray:
         Args:
             img: channel first array, must have shape: (num_channels, H[, W, ..., ]),
         """
-        flipped = []
-        for channel in img:
-            flipped.append(np.flip(channel, self.spatial_axis))
-        return np.stack(flipped).astype(img.dtype)
+
+        result: np.ndarray = np.flip(img, map_spatial_axes(img.ndim, self.spatial_axis))
+        return result.astype(img.dtype)
 
 
 class Resize(Transform):
@@ -372,7 +377,7 @@ def __call__(
             align_corners=self.align_corners if align_corners is None else align_corners,
         )
         resized = resized.squeeze(0).detach().cpu().numpy()
-        return resized
+        return np.asarray(resized)
 
 
 class Rotate(Transform):
@@ -404,7 +409,7 @@ def __init__(
         mode: Union[GridSampleMode, str] = GridSampleMode.BILINEAR,
         padding_mode: Union[GridSamplePadMode, str] = GridSamplePadMode.BORDER,
         align_corners: bool = False,
-        dtype: Optional[np.dtype] = np.float64,
+        dtype: DtypeLike = np.float64,
     ) -> None:
         self.angle = angle
         self.keep_size = keep_size
@@ -419,7 +424,7 @@ def __call__(
         mode: Optional[Union[GridSampleMode, str]] = None,
         padding_mode: Optional[Union[GridSamplePadMode, str]] = None,
         align_corners: Optional[bool] = None,
-        dtype: Optional[np.dtype] = None,
+        dtype: DtypeLike = None,
     ) -> np.ndarray:
         """
         Args:
@@ -457,7 +462,7 @@ def __call__(
                 (len(im_shape), -1)
             )
             corners = transform[:-1, :-1] @ corners
-            output_shape = (corners.ptp(axis=1) + 0.5).astype(int)
+            output_shape = np.asarray(corners.ptp(axis=1) + 0.5, dtype=int)
         shift_1 = create_translate(input_ndim, -(output_shape - 1) / 2)
         transform = shift @ transform @ shift_1
 
@@ -473,8 +478,7 @@ def __call__(
             torch.as_tensor(np.ascontiguousarray(transform).astype(_dtype)),
             spatial_size=output_shape,
         )
-        output = output.squeeze(0).detach().cpu().numpy().astype(np.float32)
-        return output
+        return np.asarray(output.squeeze(0).detach().cpu().numpy(), dtype=np.float32)
 
 
 class Zoom(Transform):
@@ -522,7 +526,7 @@ def __call__(
         mode: Optional[Union[InterpolateMode, str]] = None,
         padding_mode: Optional[Union[NumpyPadMode, str]] = None,
         align_corners: Optional[bool] = None,
-    ) -> np.ndarray:
+    ):
         """
         Args:
             img: channel first array, must have shape: (num_channels, H[, W, ..., ]).
@@ -568,6 +572,9 @@ def __call__(
 class Rotate90(Transform):
     """
     Rotate an array by 90 degrees in the plane specified by `axes`.
+    See np.rot90 for additional details:
+    https://numpy.org/doc/stable/reference/generated/numpy.rot90.html.
+
     """
 
     def __init__(self, k: int = 1, spatial_axes: Tuple[int, int] = (0, 1)) -> None:
@@ -576,6 +583,7 @@ def __init__(self, k: int = 1, spatial_axes: Tuple[int, int] = (0, 1)) -> None:
             k: number of times to rotate by 90 degrees.
             spatial_axes: 2 int numbers, defines the plane to rotate with 2 spatial axes.
                 Default: (0, 1), this is the first two axis in spatial dimensions.
+                If axis is negative it counts from the last to the first axis.
         """
         self.k = k
         spatial_axes_ = ensure_tuple(spatial_axes)
@@ -588,10 +596,9 @@ def __call__(self, img: np.ndarray) -> np.ndarray:
         Args:
             img: channel first array, must have shape: (num_channels, H[, W, ..., ]),
         """
-        rotated = []
-        for channel in img:
-            rotated.append(np.rot90(channel, self.k, self.spatial_axes))
-        return np.stack(rotated).astype(img.dtype)
+
+        result: np.ndarray = np.rot90(img, self.k, map_spatial_axes(img.ndim, self.spatial_axes))
+        return result.astype(img.dtype)
 
 
 class RandRotate90(Randomizable, Transform):
@@ -670,7 +677,7 @@ def __init__(
         mode: Union[GridSampleMode, str] = GridSampleMode.BILINEAR,
         padding_mode: Union[GridSamplePadMode, str] = GridSamplePadMode.BORDER,
         align_corners: bool = False,
-        dtype: Optional[np.dtype] = np.float64,
+        dtype: DtypeLike = np.float64,
     ) -> None:
         self.range_x = ensure_tuple(range_x)
         if len(self.range_x) == 1:
@@ -706,7 +713,7 @@ def __call__(
         mode: Optional[Union[GridSampleMode, str]] = None,
         padding_mode: Optional[Union[GridSamplePadMode, str]] = None,
         align_corners: Optional[bool] = None,
-        dtype: Optional[np.dtype] = None,
+        dtype: DtypeLike = None,
     ) -> np.ndarray:
         """
         Args:
@@ -856,12 +863,15 @@ def __call__(
             # if 2 zoom factors provided for 3D data, use the first factor for H and W dims, second factor for D dim
             self._zoom = ensure_tuple_rep(self._zoom[0], img.ndim - 2) + ensure_tuple(self._zoom[-1])
         zoomer = Zoom(self._zoom, keep_size=self.keep_size)
-        return zoomer(
-            img,
-            mode=mode or self.mode,
-            padding_mode=padding_mode or self.padding_mode,
-            align_corners=self.align_corners if align_corners is None else align_corners,
-        ).astype(_dtype)
+        return np.asarray(
+            zoomer(
+                img,
+                mode=mode or self.mode,
+                padding_mode=padding_mode or self.padding_mode,
+                align_corners=self.align_corners if align_corners is None else align_corners,
+            ),
+            dtype=_dtype,
+        )
 
 
 class AffineGrid(Transform):
@@ -937,13 +947,15 @@ def __call__(
             affine = affine @ create_scale(spatial_dims, self.scale_params)
         affine = torch.as_tensor(np.ascontiguousarray(affine), device=self.device)
 
-        grid = torch.tensor(grid) if not torch.is_tensor(grid) else grid.detach().clone()
+        grid = torch.tensor(grid) if not isinstance(grid, torch.Tensor) else grid.detach().clone()
         if self.device:
             grid = grid.to(self.device)
         grid = (affine.float() @ grid.reshape((grid.shape[0], -1)).float()).reshape([-1] + list(grid.shape[1:]))
+        if grid is None or not isinstance(grid, torch.Tensor):
+            raise ValueError("Unknown grid.")
         if self.as_tensor_output:
             return grid
-        return grid.cpu().numpy()
+        return np.asarray(grid.cpu().numpy())
 
 
 class RandAffineGrid(Randomizable, Transform):
@@ -1069,7 +1081,7 @@ def randomize(self, grid_size: Sequence[int]) -> None:
         self.random_offset = self.R.normal(size=([len(grid_size)] + list(grid_size))).astype(np.float32)
         self.rand_mag = self.R.uniform(self.magnitude[0], self.magnitude[1])
 
-    def __call__(self, spatial_size: Sequence[int]) -> Union[np.ndarray, torch.Tensor]:
+    def __call__(self, spatial_size: Sequence[int]):
         """
         Args:
             spatial_size: spatial size of the grid.
@@ -1129,11 +1141,11 @@ def __call__(
                 See also: https://pytorch.org/docs/stable/nn.functional.html#grid-sample
         """
 
-        if not torch.is_tensor(img):
+        if not isinstance(img, torch.Tensor):
             img = torch.as_tensor(np.ascontiguousarray(img))
         if grid is None:
             raise AssertionError("Error, grid argument must be supplied as an ndarray or tensor ")
-        grid = torch.tensor(grid) if not torch.is_tensor(grid) else grid.detach().clone()
+        grid = torch.tensor(grid) if not isinstance(grid, torch.Tensor) else grid.detach().clone()
         if self.device:
             img = img.to(self.device)
             grid = grid.to(self.device)
@@ -1173,8 +1185,8 @@ def __call__(
                 align_corners=True,
             )[0]
         if self.as_tensor_output:
-            return out
-        return out.cpu().numpy()
+            return torch.as_tensor(out)
+        return np.asarray(out.cpu().numpy())
 
 
 class Affine(Transform):
@@ -1499,12 +1511,12 @@ def __call__(
             grid = self.rand_affine_grid(grid=grid)
             grid = torch.nn.functional.interpolate(  # type: ignore
                 recompute_scale_factor=True,
-                input=grid.unsqueeze(0),
+                input=torch.as_tensor(grid).unsqueeze(0),
                 scale_factor=list(ensure_tuple(self.deform_grid.spacing)),
                 mode=InterpolateMode.BICUBIC.value,
                 align_corners=False,
             )
-            grid = CenterSpatialCrop(roi_size=sp_size)(grid[0])
+            grid = CenterSpatialCrop(roi_size=sp_size)(np.asarray(grid[0]))
         else:
             grid = create_grid(spatial_size=sp_size)
         return self.resampler(img, grid, mode=mode or self.mode, padding_mode=padding_mode or self.padding_mode)
diff --git a/monai/transforms/spatial/dictionary.py b/monai/transforms/spatial/dictionary.py
index 615a327d90..e612a25ef8 100644
--- a/monai/transforms/spatial/dictionary.py
+++ b/monai/transforms/spatial/dictionary.py
@@ -20,7 +20,7 @@
 import numpy as np
 import torch
 
-from monai.config import KeysCollection
+from monai.config import DtypeLike, KeysCollection
 from monai.networks.layers.simplelayers import GaussianFilter
 from monai.transforms.compose import MapTransform, Randomizable
 from monai.transforms.croppad.array import CenterSpatialCrop
@@ -120,7 +120,7 @@ def __init__(
         mode: GridSampleModeSequence = GridSampleMode.BILINEAR,
         padding_mode: GridSamplePadModeSequence = GridSamplePadMode.BORDER,
         align_corners: Union[Sequence[bool], bool] = False,
-        dtype: Optional[Union[Sequence[np.dtype], np.dtype]] = np.float64,
+        dtype: Optional[Union[Sequence[DtypeLike], DtypeLike]] = np.float64,
         meta_key_postfix: str = "meta_dict",
     ) -> None:
         """
@@ -152,7 +152,7 @@ def __init__(
             dtype: data type for resampling computation. Defaults to ``np.float64`` for best precision.
                 If None, use the data type of input data. To be compatible with other modules,
                 the output data type is always ``np.float32``.
-                It also can be a sequence of np.dtype, each element corresponds to a key in ``keys``.
+                It also can be a sequence of dtypes, each element corresponds to a key in ``keys``.
             meta_key_postfix: use `key_{postfix}` to to fetch the meta data according to the key data,
                 default is `meta_dict`, the meta data is a dictionary object.
                 For example, to handle key `image`,  read/write affine matrices from the
@@ -175,13 +175,13 @@ def __init__(
     def __call__(
         self, data: Mapping[Union[Hashable, str], Dict[str, np.ndarray]]
     ) -> Dict[Union[Hashable, str], Union[np.ndarray, Dict[str, np.ndarray]]]:
-        d = dict(data)
+        d: Dict = dict(data)
         for idx, key in enumerate(self.keys):
             meta_data = d[f"{key}_{self.meta_key_postfix}"]
             # resample array of each corresponding key
             # using affine fetched from d[affine_key]
             d[key], _, new_affine = self.spacing_transform(
-                data_array=d[key],
+                data_array=np.asarray(d[key]),
                 affine=meta_data["affine"],
                 mode=self.mode[idx],
                 padding_mode=self.padding_mode[idx],
@@ -244,7 +244,7 @@ def __init__(
     def __call__(
         self, data: Mapping[Union[Hashable, str], Dict[str, np.ndarray]]
     ) -> Dict[Union[Hashable, str], Union[np.ndarray, Dict[str, np.ndarray]]]:
-        d = dict(data)
+        d: Dict = dict(data)
         for key in self.keys:
             meta_data = d[f"{key}_{self.meta_key_postfix}"]
             d[key], _, new_affine = self.ornt_transform(d[key], affine=meta_data["affine"])
@@ -796,7 +796,7 @@ def __init__(
         mode: GridSampleModeSequence = GridSampleMode.BILINEAR,
         padding_mode: GridSamplePadModeSequence = GridSamplePadMode.BORDER,
         align_corners: Union[Sequence[bool], bool] = False,
-        dtype: Union[Sequence[Optional[np.dtype]], Optional[np.dtype]] = np.float64,
+        dtype: Union[Sequence[DtypeLike], DtypeLike] = np.float64,
     ) -> None:
         super().__init__(keys)
         self.rotator = Rotate(angle=angle, keep_size=keep_size)
@@ -864,7 +864,7 @@ def __init__(
         mode: GridSampleModeSequence = GridSampleMode.BILINEAR,
         padding_mode: GridSamplePadModeSequence = GridSamplePadMode.BORDER,
         align_corners: Union[Sequence[bool], bool] = False,
-        dtype: Union[Sequence[Optional[np.dtype]], Optional[np.dtype]] = np.float64,
+        dtype: Union[Sequence[DtypeLike], DtypeLike] = np.float64,
     ) -> None:
         super().__init__(keys)
         self.range_x = ensure_tuple(range_x)
diff --git a/monai/transforms/utility/array.py b/monai/transforms/utility/array.py
index 5476e800f4..c0ae40de59 100644
--- a/monai/transforms/utility/array.py
+++ b/monai/transforms/utility/array.py
@@ -15,11 +15,12 @@
 
 import logging
 import time
-from typing import Callable, List, Optional, Sequence, Tuple, TypeVar, Union
+from typing import Callable, List, Optional, Sequence, Tuple, Union
 
 import numpy as np
 import torch
 
+from monai.config import DtypeLike, NdarrayTensor
 from monai.transforms.compose import Randomizable, Transform
 from monai.transforms.utils import extreme_points_to_image, get_extreme_points, map_binary_to_indices
 from monai.utils import ensure_tuple, min_version, optional_import
@@ -46,10 +47,6 @@
     "TorchVision",
 ]
 
-# Generic type which can represent either a numpy.ndarray or a torch.Tensor
-# Unlike Union can create a dependence between parameter(s) / return(s)
-NdarrayTensor = TypeVar("NdarrayTensor", np.ndarray, torch.Tensor)
-
 
 class Identity(Transform):
     """
@@ -135,7 +132,7 @@ class AddChannel(Transform):
     transforms.
     """
 
-    def __call__(self, img: NdarrayTensor) -> NdarrayTensor:
+    def __call__(self, img: NdarrayTensor):
         """
         Apply the transform to `img`.
         """
@@ -209,7 +206,7 @@ class CastToType(Transform):
     specified PyTorch data type.
     """
 
-    def __init__(self, dtype: Union[np.dtype, torch.dtype] = np.float32) -> None:
+    def __init__(self, dtype=np.float32) -> None:
         """
         Args:
             dtype: convert image to this data type, default is `np.float32`.
@@ -217,7 +214,7 @@ def __init__(self, dtype: Union[np.dtype, torch.dtype] = np.float32) -> None:
         self.dtype = dtype
 
     def __call__(
-        self, img: Union[np.ndarray, torch.Tensor], dtype: Optional[Union[np.dtype, torch.dtype]] = None
+        self, img: Union[np.ndarray, torch.Tensor], dtype: Optional[Union[DtypeLike, torch.dtype]] = None
     ) -> Union[np.ndarray, torch.Tensor]:
         """
         Apply the transform to `img`, assuming `img` is a numpy array or PyTorch Tensor.
@@ -230,8 +227,8 @@ def __call__(
 
         """
         if isinstance(img, np.ndarray):
-            return img.astype(self.dtype if dtype is None else dtype)
-        if torch.is_tensor(img):
+            return img.astype(self.dtype if dtype is None else dtype)  # type: ignore
+        if isinstance(img, torch.Tensor):
             return torch.as_tensor(img, dtype=self.dtype if dtype is None else dtype)
         raise TypeError(f"img must be one of (numpy.ndarray, torch.Tensor) but is {type(img).__name__}.")
 
@@ -245,7 +242,7 @@ def __call__(self, img: Union[np.ndarray, torch.Tensor]) -> torch.Tensor:
         """
         Apply the transform to `img` and make it contiguous.
         """
-        if torch.is_tensor(img):
+        if isinstance(img, torch.Tensor):
             return img.contiguous()
         return torch.as_tensor(np.ascontiguousarray(img))
 
@@ -259,7 +256,7 @@ def __call__(self, img: Union[List, Tuple, np.ndarray, torch.Tensor]) -> np.ndar
         """
         Apply the transform to `img` and make it contiguous.
         """
-        if torch.is_tensor(img):
+        if isinstance(img, torch.Tensor):
             img = img.detach().cpu().numpy()  # type: ignore
         return np.ascontiguousarray(img)
 
@@ -276,7 +273,7 @@ def __call__(self, img: np.ndarray) -> np.ndarray:
         """
         Apply the transform to `img`.
         """
-        return img.transpose(self.indices)
+        return img.transpose(self.indices)  # type: ignore
 
 
 class SqueezeDim(Transform):
@@ -303,7 +300,7 @@ def __call__(self, img: NdarrayTensor) -> NdarrayTensor:
         Args:
             img: numpy arrays with required dimension `dim` removed
         """
-        return img.squeeze(self.dim)
+        return img.squeeze(self.dim)  # type: ignore
 
 
 class DataStats(Transform):
@@ -372,7 +369,7 @@ def __call__(
         if self.value_range if value_range is None else value_range:
             if isinstance(img, np.ndarray):
                 lines.append(f"Value range: ({np.min(img)}, {np.max(img)})")
-            elif torch.is_tensor(img):
+            elif isinstance(img, torch.Tensor):
                 lines.append(f"Value range: ({torch.min(img)}, {torch.max(img)})")
             else:
                 lines.append(f"Value range: (not a PyTorch or Numpy array, type: {type(img)})")
@@ -497,7 +494,7 @@ def __init__(  # pytype: disable=annotation-type-mismatch
 
     def __call__(
         self, img: np.ndarray, select_labels: Optional[Union[Sequence[int], int]] = None, merge_channels: bool = False
-    ) -> np.ndarray:
+    ):
         """
         Args:
             select_labels: labels to generate mask from. for 1 channel label, the `select_labels`
@@ -617,7 +614,7 @@ def __call__(
         sigma: Union[Sequence[float], float, Sequence[torch.Tensor], torch.Tensor] = 3.0,
         rescale_min: float = -1.0,
         rescale_max: float = 1.0,
-    ) -> np.ndarray:
+    ):
         """
         Args:
             img: the image that we want to add new channel to.
diff --git a/monai/transforms/utility/dictionary.py b/monai/transforms/utility/dictionary.py
index ef89dbe32d..f374b82d76 100644
--- a/monai/transforms/utility/dictionary.py
+++ b/monai/transforms/utility/dictionary.py
@@ -17,12 +17,12 @@
 
 import copy
 import logging
-from typing import Callable, Dict, Hashable, List, Mapping, Optional, Sequence, Tuple, Union
+from typing import Any, Callable, Dict, Hashable, List, Mapping, Optional, Sequence, Tuple, Union
 
 import numpy as np
 import torch
 
-from monai.config import KeysCollection
+from monai.config import DtypeLike, KeysCollection, NdarrayTensor
 from monai.transforms.compose import MapTransform, Randomizable
 from monai.transforms.utility.array import (
     AddChannel,
@@ -64,10 +64,12 @@
     "CopyItemsd",
     "ConcatItemsd",
     "Lambdad",
+    "RandLambdad",
     "LabelToMaskd",
     "FgBgToIndicesd",
     "ConvertToMultiChannelBasedOnBratsClassesd",
     "AddExtremePointsChanneld",
+    "TorchVisiond",
     "IdentityD",
     "IdentityDict",
     "AsChannelFirstD",
@@ -76,6 +78,8 @@
     "AsChannelLastDict",
     "AddChannelD",
     "AddChannelDict",
+    "RandLambdaD",
+    "RandLambdaDict",
     "RepeatChannelD",
     "RepeatChannelDict",
     "SplitChannelD",
@@ -106,7 +110,6 @@
     "ConvertToMultiChannelBasedOnBratsClassesDict",
     "AddExtremePointsChannelD",
     "AddExtremePointsChannelDict",
-    "TorchVisiond",
     "TorchVisionD",
     "TorchVisionDict",
 ]
@@ -127,7 +130,9 @@ def __init__(self, keys: KeysCollection) -> None:
         super().__init__(keys)
         self.identity = Identity()
 
-    def __call__(self, data: Mapping[Hashable, Union[np.ndarray, torch.Tensor]]) -> Dict[Hashable, np.ndarray]:
+    def __call__(
+        self, data: Mapping[Hashable, Union[np.ndarray, torch.Tensor]]
+    ) -> Dict[Hashable, Union[np.ndarray, torch.Tensor]]:
         d = dict(data)
         for key in self.keys:
             d[key] = self.identity(d[key])
@@ -192,9 +197,7 @@ def __init__(self, keys: KeysCollection) -> None:
         super().__init__(keys)
         self.adder = AddChannel()
 
-    def __call__(
-        self, data: Mapping[Hashable, Union[np.ndarray, torch.Tensor]]
-    ) -> Dict[Hashable, Union[np.ndarray, torch.Tensor]]:
+    def __call__(self, data: Mapping[Hashable, NdarrayTensor]) -> Dict[Hashable, NdarrayTensor]:
         d = dict(data)
         for key in self.keys:
             d[key] = self.adder(d[key])
@@ -279,14 +282,14 @@ class CastToTyped(MapTransform):
     def __init__(
         self,
         keys: KeysCollection,
-        dtype: Union[Sequence[Union[np.dtype, torch.dtype]], np.dtype, torch.dtype] = np.float32,
+        dtype: Union[Sequence[Union[DtypeLike, torch.dtype]], DtypeLike, torch.dtype] = np.float32,
     ) -> None:
         """
         Args:
             keys: keys of the corresponding items to be transformed.
                 See also: :py:class:`monai.transforms.compose.MapTransform`
             dtype: convert image to this data type, default is `np.float32`.
-                it also can be a sequence of np.dtype or torch.dtype,
+                it also can be a sequence of dtypes or torch.dtype,
                 each element corresponds to a key in ``keys``.
 
         """
@@ -318,7 +321,9 @@ def __init__(self, keys: KeysCollection) -> None:
         super().__init__(keys)
         self.converter = ToTensor()
 
-    def __call__(self, data: Mapping[Hashable, Union[np.ndarray, torch.Tensor]]) -> Dict[Hashable, torch.Tensor]:
+    def __call__(
+        self, data: Mapping[Hashable, Union[np.ndarray, torch.Tensor]]
+    ) -> Dict[Hashable, Union[np.ndarray, torch.Tensor]]:
         d = dict(data)
         for key in self.keys:
             d[key] = self.converter(d[key])
@@ -339,7 +344,9 @@ def __init__(self, keys: KeysCollection) -> None:
         super().__init__(keys)
         self.converter = ToNumpy()
 
-    def __call__(self, data: Mapping[Hashable, Union[np.ndarray, torch.Tensor]]) -> Dict[Hashable, np.ndarray]:
+    def __call__(
+        self, data: Mapping[Hashable, Union[np.ndarray, torch.Tensor]]
+    ) -> Dict[Hashable, Union[np.ndarray, torch.Tensor]]:
         d = dict(data)
         for key in self.keys:
             d[key] = self.converter(d[key])
@@ -382,9 +389,7 @@ def __init__(self, keys: KeysCollection, dim: int = 0) -> None:
         super().__init__(keys)
         self.converter = SqueezeDim(dim=dim)
 
-    def __call__(
-        self, data: Mapping[Hashable, Union[np.ndarray, torch.Tensor]]
-    ) -> Dict[Hashable, Union[np.ndarray, torch.Tensor]]:
+    def __call__(self, data: Mapping[Hashable, NdarrayTensor]) -> Dict[Hashable, NdarrayTensor]:
         d = dict(data)
         for key in self.keys:
             d[key] = self.converter(d[key])
@@ -435,9 +440,7 @@ def __init__(
         self.logger_handler = logger_handler
         self.printer = DataStats(logger_handler=logger_handler)
 
-    def __call__(
-        self, data: Mapping[Hashable, Union[np.ndarray, torch.Tensor]]
-    ) -> Dict[Hashable, Union[np.ndarray, torch.Tensor]]:
+    def __call__(self, data: Mapping[Hashable, NdarrayTensor]) -> Dict[Hashable, NdarrayTensor]:
         d = dict(data)
         for idx, key in enumerate(self.keys):
             d[key] = self.printer(
@@ -469,9 +472,7 @@ def __init__(self, keys: KeysCollection, delay_time: Union[Sequence[float], floa
         self.delay_time = ensure_tuple_rep(delay_time, len(self.keys))
         self.delayer = SimulateDelay()
 
-    def __call__(
-        self, data: Mapping[Hashable, Union[np.ndarray, torch.Tensor]]
-    ) -> Dict[Hashable, Union[np.ndarray, torch.Tensor]]:
+    def __call__(self, data: Mapping[Hashable, NdarrayTensor]) -> Dict[Hashable, NdarrayTensor]:
         d = dict(data)
         for idx, key in enumerate(self.keys):
             d[key] = self.delayer(d[key], delay_time=self.delay_time[idx])
@@ -623,6 +624,27 @@ def __call__(self, data):
         return d
 
 
+class RandLambdad(Lambdad, Randomizable):
+    """
+    Randomizable version :py:class:`monai.transforms.Lambdad`, the input `func` contains random logic.
+    It's a randomizable transform so `CacheDataset` will not execute it and cache the results.
+
+    Args:
+        keys: keys of the corresponding items to be transformed.
+            See also: :py:class:`monai.transforms.compose.MapTransform`
+        func: Lambda/function to be applied. It also can be a sequence of Callable,
+            each element corresponds to a key in ``keys``.
+        overwrite: whether to overwrite the original data in the input dictionary with lamdbda function output.
+            default to True. it also can be a sequence of bool, each element corresponds to a key in ``keys``.
+
+    For more details, please check :py:class:`monai.transforms.Lambdad`.
+
+    """
+
+    def randomize(self, data: Any) -> None:
+        pass
+
+
 class LabelToMaskd(MapTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.LabelToMask`.
@@ -832,3 +854,4 @@ def __call__(self, data: Mapping[Hashable, torch.Tensor]) -> Dict[Hashable, torc
 ) = ConvertToMultiChannelBasedOnBratsClassesd
 AddExtremePointsChannelD = AddExtremePointsChannelDict = AddExtremePointsChanneld
 TorchVisionD = TorchVisionDict = TorchVisiond
+RandLambdaD = RandLambdaDict = RandLambdad
diff --git a/monai/transforms/utils.py b/monai/transforms/utils.py
index 23c6bd100a..9a84eb00d9 100644
--- a/monai/transforms/utils.py
+++ b/monai/transforms/utils.py
@@ -17,7 +17,7 @@
 import numpy as np
 import torch
 
-from monai.config import IndexSelection
+from monai.config import DtypeLike, IndexSelection
 from monai.networks.layers import GaussianFilter
 from monai.utils import ensure_tuple, ensure_tuple_rep, ensure_tuple_size, fall_back_tuple, min_version, optional_import
 
@@ -48,6 +48,7 @@
     "get_largest_connected_component_mask",
     "get_extreme_points",
     "extreme_points_to_image",
+    "map_spatial_axes",
 ]
 
 
@@ -58,7 +59,7 @@ def rand_choice(prob: float = 0.5) -> bool:
     return bool(random.random() <= prob)
 
 
-def img_bounds(img: np.ndarray) -> np.ndarray:
+def img_bounds(img: np.ndarray):
     """
     Returns the minimum and maximum indices of non-zero lines in axis 0 of `img`, followed by that for axis 1.
     """
@@ -91,9 +92,7 @@ def zero_margins(img: np.ndarray, margin: int) -> bool:
     return not np.any(img[:, :margin, :]) and not np.any(img[:, -margin:, :])
 
 
-def rescale_array(
-    arr: np.ndarray, minv: float = 0.0, maxv: float = 1.0, dtype: Optional[np.dtype] = np.float32
-) -> np.ndarray:
+def rescale_array(arr: np.ndarray, minv: float = 0.0, maxv: float = 1.0, dtype: DtypeLike = np.float32):
     """
     Rescale the values of numpy array `arr` to be from `minv` to `maxv`.
     """
@@ -111,7 +110,7 @@ def rescale_array(
 
 
 def rescale_instance_array(
-    arr: np.ndarray, minv: float = 0.0, maxv: float = 1.0, dtype: np.dtype = np.float32
+    arr: np.ndarray, minv: float = 0.0, maxv: float = 1.0, dtype: DtypeLike = np.float32
 ) -> np.ndarray:
     """
     Rescale each array slice along the first dimension of `arr` independently.
@@ -123,12 +122,12 @@ def rescale_instance_array(
     return out
 
 
-def rescale_array_int_max(arr: np.ndarray, dtype: np.dtype = np.uint16) -> np.ndarray:
+def rescale_array_int_max(arr: np.ndarray, dtype: DtypeLike = np.uint16) -> np.ndarray:
     """
     Rescale the array `arr` to be between the minimum and maximum values of the type `dtype`.
     """
     info: np.iinfo = np.iinfo(dtype)
-    return rescale_array(arr, info.min, info.max).astype(dtype)
+    return np.asarray(rescale_array(arr, info.min, info.max), dtype=dtype)
 
 
 def copypaste_arrays(
@@ -191,9 +190,7 @@ def copypaste_arrays(
     return tuple(srcslices), tuple(destslices)
 
 
-def resize_center(
-    img: np.ndarray, *resize_dims: Optional[int], fill_value: float = 0.0, inplace: bool = True
-) -> np.ndarray:
+def resize_center(img: np.ndarray, *resize_dims: Optional[int], fill_value: float = 0.0, inplace: bool = True):
     """
     Resize `img` by cropping or expanding the image from the center. The `resize_dims` values are the output dimensions
     (or None to use original dimension of `img`). If a dimension is smaller than that of `img` then the result will be
@@ -208,7 +205,7 @@ def resize_center(
     srcslices, destslices = copypaste_arrays(img.shape, resize_dims, half_img_shape, half_dest_shape, resize_dims)
 
     if not inplace:
-        dest = np.full(resize_dims, fill_value, img.dtype)
+        dest = np.full(resize_dims, fill_value, img.dtype)  # type: ignore
         dest[destslices] = img[srcslices]
         return dest
     return img[srcslices]
@@ -271,8 +268,8 @@ def weighted_patch_samples(
         raise ValueError("w must be an ND array.")
     if r_state is None:
         r_state = np.random.RandomState()
-    img_size = np.asarray(w.shape, dtype=np.int)
-    win_size = np.asarray(fall_back_tuple(spatial_size, img_size), dtype=np.int)
+    img_size = np.asarray(w.shape, dtype=int)
+    win_size = np.asarray(fall_back_tuple(spatial_size, img_size), dtype=int)
 
     s = tuple(slice(w // 2, m - w + w // 2) if m > w else slice(m // 2, m // 2 + 1) for w, m in zip(win_size, img_size))
     v = w[s]  # weight map in the 'valid' mode
@@ -287,7 +284,7 @@ def weighted_patch_samples(
         idx = v.searchsorted(r_state.random(n_samples) * v[-1], side="right")
     # compensate 'valid' mode
     diff = np.minimum(win_size, img_size) // 2
-    return [np.unravel_index(i, v_size) + diff for i in np.asarray(idx, dtype=np.int)]
+    return [np.unravel_index(i, v_size) + diff for i in np.asarray(idx, dtype=int)]
 
 
 def generate_pos_neg_label_crop_centers(
@@ -395,8 +392,8 @@ def create_grid(
     spatial_size: Sequence[int],
     spacing: Optional[Sequence[float]] = None,
     homogeneous: bool = True,
-    dtype: np.dtype = float,
-) -> np.ndarray:
+    dtype: DtypeLike = float,
+):
     """
     compute a `spatial_size` mesh.
 
@@ -415,8 +412,8 @@ def create_grid(
 
 
 def create_control_grid(
-    spatial_shape: Sequence[int], spacing: Sequence[float], homogeneous: bool = True, dtype: np.dtype = float
-) -> np.ndarray:
+    spatial_shape: Sequence[int], spacing: Sequence[float], homogeneous: bool = True, dtype: DtypeLike = float
+):
     """
     control grid with two additional point in each direction
     """
@@ -461,11 +458,15 @@ def create_rotate(spatial_dims: int, radians: Union[Sequence[float], float]) ->
             )
         if len(radians) >= 2:
             sin_, cos_ = np.sin(radians[1]), np.cos(radians[1])
+            if affine is None:
+                raise ValueError("Affine should be a matrix.")
             affine = affine @ np.array(
                 [[cos_, 0.0, sin_, 0.0], [0.0, 1.0, 0.0, 0.0], [-sin_, 0.0, cos_, 0.0], [0.0, 0.0, 0.0, 1.0]]
             )
         if len(radians) >= 3:
             sin_, cos_ = np.sin(radians[2]), np.cos(radians[2])
+            if affine is None:
+                raise ValueError("Affine should be a matrix.")
             affine = affine @ np.array(
                 [[cos_, -sin_, 0.0, 0.0], [sin_, cos_, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]
             )
@@ -504,7 +505,7 @@ def create_shear(spatial_dims: int, coefs: Union[Sequence[float], float]) -> np.
     raise NotImplementedError("Currently only spatial_dims in [2, 3] are supported.")
 
 
-def create_scale(spatial_dims: int, scaling_factor: Union[Sequence[float], float]) -> np.ndarray:
+def create_scale(spatial_dims: int, scaling_factor: Union[Sequence[float], float]):
     """
     create a scaling matrix
 
@@ -528,7 +529,7 @@ def create_translate(spatial_dims: int, shift: Union[Sequence[float], float]) ->
     affine = np.eye(spatial_dims + 1)
     for i, a in enumerate(shift[:spatial_dims]):
         affine[i, spatial_dims] = a
-    return affine
+    return np.asarray(affine)
 
 
 def generate_spatial_bounding_box(
@@ -690,3 +691,42 @@ def extreme_points_to_image(
     points_image = (points_image - min_intensity) / (max_intensity - min_intensity)
     points_image = points_image * (rescale_max - rescale_min) + rescale_min
     return points_image
+
+
+def map_spatial_axes(
+    img_ndim: int,
+    spatial_axes: Optional[Union[Sequence[int], int]] = None,
+    channel_first: bool = True,
+) -> List[int]:
+    """
+    Utility to map the spatial axes to real axes in channel first/last shape.
+    For example:
+    If `channel_first` is True, and `img` has 3 spatial dims, map spatial axes to real axes as below:
+    None -> [1, 2, 3]
+    [0, 1] -> [1, 2]
+    [0, -1] -> [1, -1]
+    If `channel_first` is False, and `img` has 3 spatial dims, map spatial axes to real axes as below:
+    None -> [0, 1, 2]
+    [0, 1] -> [0, 1]
+    [0, -1] -> [0, -2]
+
+    Args:
+        img_ndim: dimension number of the target image.
+        spatial_axes: spatial axes to be converted, default is None.
+            The default `None` will convert to all the spatial axes of the image.
+            If axis is negative it counts from the last to the first axis.
+            If axis is a tuple of ints.
+        channel_first: the image data is channel first or channel last, defaut to channel first.
+
+    """
+    if spatial_axes is None:
+        spatial_axes_ = list(range(1, img_ndim) if channel_first else range(0, img_ndim - 1))
+    else:
+        spatial_axes_ = []
+        for a in ensure_tuple(spatial_axes):
+            if channel_first:
+                spatial_axes_.append(a if a < 0 else a + 1)
+            else:
+                spatial_axes_.append(a - 1 if a < 0 else a)
+
+    return spatial_axes_
diff --git a/monai/utils/__init__.py b/monai/utils/__init__.py
index e5567f9f16..1e17d44029 100644
--- a/monai/utils/__init__.py
+++ b/monai/utils/__init__.py
@@ -32,6 +32,7 @@
 )
 from .misc import (
     MAX_SEED,
+    ImageMetaKey,
     copy_to_device,
     dtype_numpy_to_torch,
     dtype_torch_to_numpy,
diff --git a/monai/utils/misc.py b/monai/utils/misc.py
index 2b31392a46..f9346340cf 100644
--- a/monai/utils/misc.py
+++ b/monai/utils/misc.py
@@ -41,6 +41,7 @@
     "dtype_numpy_to_torch",
     "MAX_SEED",
     "copy_to_device",
+    "ImageMetaKey",
 ]
 
 _seed = None
@@ -76,7 +77,7 @@ def issequenceiterable(obj: Any) -> bool:
     """
     Determine if the object is an iterable sequence and is not a string.
     """
-    if torch.is_tensor(obj):
+    if isinstance(obj, torch.Tensor):
         return int(obj.dim()) > 0  # a 0-d tensor is not iterable
     return isinstance(obj, collections.abc.Iterable) and not isinstance(obj, str)
 
@@ -130,7 +131,9 @@ def ensure_tuple_rep(tup: Any, dim: int) -> Tuple[Any, ...]:
     raise ValueError(f"Sequence must have length {dim}, got {len(tup)}.")
 
 
-def fall_back_tuple(user_provided: Any, default: Sequence, func: Callable = lambda x: x and x > 0) -> Tuple[Any, ...]:
+def fall_back_tuple(
+    user_provided: Any, default: Union[Sequence, np.ndarray], func: Callable = lambda x: x and x > 0
+) -> Tuple[Any, ...]:
     """
     Refine `user_provided` according to the `default`, and returns as a validated tuple.
 
@@ -175,13 +178,13 @@ def fall_back_tuple(user_provided: Any, default: Sequence, func: Callable = lamb
 
 
 def is_scalar_tensor(val: Any) -> bool:
-    if torch.is_tensor(val) and val.ndim == 0:
+    if isinstance(val, torch.Tensor) and val.ndim == 0:
         return True
     return False
 
 
 def is_scalar(val: Any) -> bool:
-    if torch.is_tensor(val) and val.ndim == 0:
+    if isinstance(val, torch.Tensor) and val.ndim == 0:
         return True
     return bool(np.isscalar(val))
 
@@ -287,7 +290,7 @@ def _parse_var(s):
 
 
 _torch_to_np_dtype = {
-    torch.bool: np.bool,
+    torch.bool: bool,
     torch.uint8: np.uint8,
     torch.int8: np.int8,
     torch.int16: np.int16,
@@ -347,3 +350,11 @@ def copy_to_device(
         warnings.warn(f"{fn_name} called with incompatible type: " + f"{type(obj)}. Data will be returned unchanged.")
 
     return obj
+
+
+class ImageMetaKey:
+    """
+    Common key names in the meta data header of images
+    """
+
+    FILENAME_OR_OBJ = "filename_or_obj"
diff --git a/monai/visualize/img2tensorboard.py b/monai/visualize/img2tensorboard.py
index 8f6eca5482..b02a7a80ea 100644
--- a/monai/visualize/img2tensorboard.py
+++ b/monai/visualize/img2tensorboard.py
@@ -96,7 +96,7 @@ def make_animated_gif_summary(
 
     for it_i in range(min(max_out, list(image.shape)[0])):
         one_channel_img: Union[torch.Tensor, np.ndarray] = (
-            image[it_i, :, :, :].squeeze(dim=0) if torch.is_tensor(image) else image[it_i, :, :, :]
+            image[it_i, :, :, :].squeeze(dim=0) if isinstance(image, torch.Tensor) else image[it_i, :, :, :]
         )
         summary_op = _image3_animated_gif(tag + suffix.format(it_i), one_channel_img, scale_factor)
     return summary_op
@@ -182,7 +182,7 @@ def plot_2d_or_3d_image(
         max_frames: number of frames for 2D-t plot.
         tag: tag of the plotted image on TensorBoard.
     """
-    d = data[index].detach().cpu().numpy() if torch.is_tensor(data) else data[index]
+    d = data[index].detach().cpu().numpy() if isinstance(data, torch.Tensor) else data[index]
 
     if d.ndim == 2:
         d = rescale_array(d, 0, 1)
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 3de7365d16..2a43e63d73 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -3,7 +3,7 @@
 pytorch-ignite==0.4.2
 gdown>=3.6.4
 scipy
-itk
+itk>=5.0
 nibabel
 pillow
 tensorboard
diff --git a/setup.cfg b/setup.cfg
index aff62045e1..ea61eadd92 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -30,7 +30,7 @@ all =
     pytorch-ignite==0.4.2
     gdown>=3.6.4
     torchvision
-    itk
+    itk>=5.0
     tqdm>=4.47.0
 nibabel =
     nibabel
@@ -47,7 +47,7 @@ ignite =
 torchvision =
     torchvision
 itk =
-    itk
+    itk>=5.0
 tqdm =
     tqdm>=4.47.0
 lmdb =
diff --git a/tests/min_tests.py b/tests/min_tests.py
index 665ead6cc6..999a1aeaa0 100644
--- a/tests/min_tests.py
+++ b/tests/min_tests.py
@@ -103,6 +103,12 @@ def run_testsuit():
         "test_handler_metrics_saver",
         "test_handler_metrics_saver_dist",
         "test_evenly_divisible_all_gather_dist",
+        "test_handler_classification_saver_dist",
+        "test_deepgrow_transforms",
+        "test_deepgrow_interaction",
+        "test_deepgrow_dataset",
+        "test_save_image",
+        "test_save_imaged",
     ]
     assert sorted(exclude_cases) == sorted(set(exclude_cases)), f"Duplicated items in {exclude_cases}"
 
diff --git a/tests/test_affine.py b/tests/test_affine.py
index fbda818437..934473fc5c 100644
--- a/tests/test_affine.py
+++ b/tests/test_affine.py
@@ -79,8 +79,8 @@ class TestAffine(unittest.TestCase):
     def test_affine(self, input_param, input_data, expected_val):
         g = Affine(**input_param)
         result = g(**input_data)
-        self.assertEqual(torch.is_tensor(result), torch.is_tensor(expected_val))
-        if torch.is_tensor(result):
+        self.assertEqual(isinstance(result, torch.Tensor), isinstance(expected_val, torch.Tensor))
+        if isinstance(result, torch.Tensor):
             np.testing.assert_allclose(result.cpu().numpy(), expected_val.cpu().numpy(), rtol=1e-4, atol=1e-4)
         else:
             np.testing.assert_allclose(result, expected_val, rtol=1e-4, atol=1e-4)
diff --git a/tests/test_affine_grid.py b/tests/test_affine_grid.py
index c7caae29b4..2906cd18b6 100644
--- a/tests/test_affine_grid.py
+++ b/tests/test_affine_grid.py
@@ -93,8 +93,8 @@ class TestAffineGrid(unittest.TestCase):
     def test_affine_grid(self, input_param, input_data, expected_val):
         g = AffineGrid(**input_param)
         result = g(**input_data)
-        self.assertEqual(torch.is_tensor(result), torch.is_tensor(expected_val))
-        if torch.is_tensor(result):
+        self.assertEqual(isinstance(result, torch.Tensor), isinstance(expected_val, torch.Tensor))
+        if isinstance(result, torch.Tensor):
             np.testing.assert_allclose(result.cpu().numpy(), expected_val.cpu().numpy(), rtol=1e-4, atol=1e-4)
         else:
             np.testing.assert_allclose(result, expected_val, rtol=1e-4, atol=1e-4)
diff --git a/tests/test_bilateral_approx_cpu.py b/tests/test_bilateral_approx_cpu.py
index 71cf53519c..2b6088a56f 100644
--- a/tests/test_bilateral_approx_cpu.py
+++ b/tests/test_bilateral_approx_cpu.py
@@ -16,7 +16,7 @@
 from parameterized import parameterized
 
 from monai.networks.layers.filtering import BilateralFilter
-from tests.utils import skip_if_no_cpp_extention
+from tests.utils import skip_if_no_cpp_extension
 
 TEST_CASES = [
     [
@@ -360,7 +360,7 @@
 ]
 
 
-@skip_if_no_cpp_extention
+@skip_if_no_cpp_extension
 class BilateralFilterTestCaseCpuApprox(unittest.TestCase):
     @parameterized.expand(TEST_CASES)
     def test_cpu_approx(self, test_case_description, sigmas, input, expected):
diff --git a/tests/test_bilateral_approx_cuda.py b/tests/test_bilateral_approx_cuda.py
index d0515d60e5..fdaba26f72 100644
--- a/tests/test_bilateral_approx_cuda.py
+++ b/tests/test_bilateral_approx_cuda.py
@@ -16,7 +16,7 @@
 from parameterized import parameterized
 
 from monai.networks.layers.filtering import BilateralFilter
-from tests.utils import skip_if_no_cpp_extention, skip_if_no_cuda
+from tests.utils import skip_if_no_cpp_extension, skip_if_no_cuda
 
 TEST_CASES = [
     [
@@ -361,7 +361,7 @@
 
 
 @skip_if_no_cuda
-@skip_if_no_cpp_extention
+@skip_if_no_cpp_extension
 class BilateralFilterTestCaseCudaApprox(unittest.TestCase):
     @parameterized.expand(TEST_CASES)
     def test_cuda_approx(self, test_case_description, sigmas, input, expected):
diff --git a/tests/test_bilateral_precise.py b/tests/test_bilateral_precise.py
index b02f3f04df..db2ee88239 100644
--- a/tests/test_bilateral_precise.py
+++ b/tests/test_bilateral_precise.py
@@ -16,7 +16,7 @@
 from parameterized import parameterized
 
 from monai.networks.layers.filtering import BilateralFilter
-from tests.utils import skip_if_no_cpp_extention, skip_if_no_cuda
+from tests.utils import skip_if_no_cpp_extension, skip_if_no_cuda
 
 TEST_CASES = [
     [
@@ -360,7 +360,7 @@
 ]
 
 
-@skip_if_no_cpp_extention
+@skip_if_no_cpp_extension
 class BilateralFilterTestCaseCpuPrecised(unittest.TestCase):
     @parameterized.expand(TEST_CASES)
     def test_cpu_precised(self, test_case_description, sigmas, input, expected):
@@ -378,7 +378,7 @@ def test_cpu_precised(self, test_case_description, sigmas, input, expected):
 
 
 @skip_if_no_cuda
-@skip_if_no_cpp_extention
+@skip_if_no_cpp_extension
 class BilateralFilterTestCaseCudaPrecised(unittest.TestCase):
     @parameterized.expand(TEST_CASES)
     def test_cuda_precised(self, test_case_description, sigmas, input, expected):
diff --git a/tests/test_compose.py b/tests/test_compose.py
index 3585b3453c..c049044a97 100644
--- a/tests/test_compose.py
+++ b/tests/test_compose.py
@@ -156,6 +156,17 @@ def test_data_loader_2(self):
             self.assertAlmostEqual(out_1.cpu().item(), 0.131966779)
         set_determinism(None)
 
+    def test_flatten_and_len(self):
+        x = AddChannel()
+        t1 = Compose([x, x, x, x, Compose([Compose([x, x]), x, x])])
+
+        t2 = t1.flatten()
+        for t in t2.transforms:
+            self.assertNotIsInstance(t, Compose)
+
+        # test len
+        self.assertEqual(len(t1), 8)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_crop_foregroundd.py b/tests/test_crop_foregroundd.py
index f4283514de..cacf990763 100644
--- a/tests/test_crop_foregroundd.py
+++ b/tests/test_crop_foregroundd.py
@@ -65,14 +65,14 @@ def test_value(self, argments, image, expected_data):
     @parameterized.expand([TEST_CASE_1])
     def test_foreground_position(self, argments, image, _):
         result = CropForegroundd(**argments)(image)
-        self.assertListEqual(result["foreground_start_coord"], [1, 1])
-        self.assertListEqual(result["foreground_end_coord"], [4, 4])
+        np.testing.assert_allclose(result["foreground_start_coord"], np.array([1, 1]))
+        np.testing.assert_allclose(result["foreground_end_coord"], np.array([4, 4]))
 
         argments["start_coord_key"] = "test_start_coord"
         argments["end_coord_key"] = "test_end_coord"
         result = CropForegroundd(**argments)(image)
-        self.assertListEqual(result["test_start_coord"], [1, 1])
-        self.assertListEqual(result["test_end_coord"], [4, 4])
+        np.testing.assert_allclose(result["test_start_coord"], np.array([1, 1]))
+        np.testing.assert_allclose(result["test_end_coord"], np.array([4, 4]))
 
 
 if __name__ == "__main__":
diff --git a/tests/test_deepgrow_dataset.py b/tests/test_deepgrow_dataset.py
new file mode 100644
index 0000000000..e871c328a6
--- /dev/null
+++ b/tests/test_deepgrow_dataset.py
@@ -0,0 +1,57 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import tempfile
+import unittest
+
+import nibabel as nib
+import numpy as np
+
+from monai.apps.deepgrow.dataset import create_dataset
+
+
+class TestCreateDataset(unittest.TestCase):
+    def _create_data(self, tempdir):
+        affine = np.eye(4)
+        image = np.random.randint(0, 2, size=(128, 128, 40))
+        image_file = os.path.join(tempdir, "image1.nii.gz")
+        nib.save(nib.Nifti1Image(image, affine), image_file)
+
+        label = np.zeros((128, 128, 40))
+        label[0][1][0] = 1
+        label[0][1][1] = 1
+        label[0][0][2] = 1
+        label[0][1][2] = 1
+        label_file = os.path.join(tempdir, "label1.nii.gz")
+        nib.save(nib.Nifti1Image(label, affine), label_file)
+
+        return [{"image": image_file, "label": label_file}]
+
+    def test_create_dataset_2d(self):
+        with tempfile.TemporaryDirectory() as tempdir:
+            datalist = self._create_data(tempdir)
+            output_dir = os.path.join(tempdir, "2d")
+            deepgrow_datalist = create_dataset(datalist=datalist, output_dir=output_dir, dimension=2, pixdim=(1, 1))
+            self.assertEqual(len(deepgrow_datalist), 3)
+            self.assertEqual(deepgrow_datalist[0]["region"], 1)
+
+    def test_create_dataset_3d(self):
+        with tempfile.TemporaryDirectory() as tempdir:
+            datalist = self._create_data(tempdir)
+            output_dir = os.path.join(tempdir, "3d")
+            deepgrow_datalist = create_dataset(datalist=datalist, output_dir=output_dir, dimension=3, pixdim=(1, 1, 1))
+            self.assertEqual(len(deepgrow_datalist), 1)
+            self.assertEqual(deepgrow_datalist[0]["region"], 1)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_deepgrow_interaction.py b/tests/test_deepgrow_interaction.py
new file mode 100644
index 0000000000..272ae82a5b
--- /dev/null
+++ b/tests/test_deepgrow_interaction.py
@@ -0,0 +1,62 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import torch
+
+from monai.apps.deepgrow.interaction import Interaction
+from monai.data import Dataset
+from monai.engines import SupervisedTrainer
+from monai.transforms import Activationsd, Compose, ToNumpyd
+
+
+class TestInteractions(unittest.TestCase):
+    def run_interaction(self, train, compose):
+        data = []
+        for i in range(5):
+            data.append({"image": torch.tensor([float(i)]), "label": torch.tensor([float(i)])})
+        network = torch.nn.Linear(1, 1)
+        lr = 1e-3
+        opt = torch.optim.SGD(network.parameters(), lr)
+        loss = torch.nn.L1Loss()
+        dataset = Dataset(data, transform=None)
+        data_loader = torch.utils.data.DataLoader(dataset, batch_size=5)
+
+        iteration_transforms = [Activationsd(keys="pred", sigmoid=True), ToNumpyd(keys="pred")]
+        iteration_transforms = Compose(iteration_transforms) if compose else iteration_transforms
+
+        i = Interaction(transforms=iteration_transforms, train=train, max_interactions=5)
+        self.assertEqual(len(i.transforms.transforms), 2, "Mismatch in expected transforms")
+
+        # set up engine
+        engine = SupervisedTrainer(
+            device=torch.device("cpu"),
+            max_epochs=1,
+            train_data_loader=data_loader,
+            network=network,
+            optimizer=opt,
+            loss_function=loss,
+            iteration_update=i,
+        )
+
+        engine.run()
+        self.assertIsNotNone(engine.state.batch.get("probability"), "Probability is missing")
+
+    def test_train_interaction(self):
+        self.run_interaction(train=True, compose=True)
+
+    def test_val_interaction(self):
+        self.run_interaction(train=False, compose=False)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_deepgrow_transforms.py b/tests/test_deepgrow_transforms.py
new file mode 100644
index 0000000000..f534813832
--- /dev/null
+++ b/tests/test_deepgrow_transforms.py
@@ -0,0 +1,224 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+from parameterized import parameterized
+
+from monai.apps.deepgrow.transforms import (
+    AddGuidanceSignald,
+    AddInitialSeedPointd,
+    AddRandomGuidanced,
+    FindAllValidSlicesd,
+    FindDiscrepancyRegionsd,
+    SpatialCropForegroundd,
+)
+
+IMAGE = np.array([[[[1, 0, 2, 0, 1], [0, 1, 2, 1, 0], [2, 2, 3, 2, 2], [0, 1, 2, 1, 0], [1, 0, 2, 0, 1]]]])
+LABEL = np.array([[[[0, 0, 0, 0, 0], [0, 1, 0, 1, 0], [0, 0, 1, 0, 0], [0, 1, 0, 1, 0], [0, 0, 0, 0, 0]]]])
+BATCH_IMAGE = np.array([[[[[1, 0, 2, 0, 1], [0, 1, 2, 1, 0], [2, 2, 3, 2, 2], [0, 1, 2, 1, 0], [1, 0, 2, 0, 1]]]]])
+BATCH_LABEL = np.array([[[[[0, 0, 0, 0, 0], [0, 1, 0, 1, 0], [0, 0, 1, 0, 0], [0, 1, 0, 1, 0], [0, 0, 0, 0, 0]]]]])
+
+DATA_1 = {
+    "image": IMAGE,
+    "label": LABEL,
+    "image_meta_dict": {},
+    "label_meta_dict": {},
+}
+
+DATA_2 = {
+    "image": np.array(
+        [
+            [
+                [[1, 2, 3, 2, 1], [1, 1, 3, 2, 1], [0, 0, 0, 0, 0], [1, 1, 1, 2, 1], [0, 2, 2, 2, 1]],
+                [[1, 0, 2, 0, 1], [0, 1, 2, 1, 0], [2, 2, 3, 2, 2], [0, 1, 2, 1, 0], [1, 0, 2, 0, 1]],
+            ]
+        ]
+    ),
+    "label": np.array(
+        [
+            [
+                [[0, 0, 1, 0, 0], [0, 1, 1, 1, 0], [1, 1, 1, 1, 1], [0, 1, 1, 1, 0], [0, 0, 1, 0, 0]],
+                [[0, 0, 0, 0, 0], [0, 1, 0, 1, 0], [0, 0, 1, 0, 0], [0, 1, 0, 1, 0], [0, 0, 0, 0, 0]],
+            ]
+        ]
+    ),
+    "guidance": np.array([[[1, 0, 2, 2], [1, 1, 2, 2]], [[-1, -1, -1, -1], [-1, -1, -1, -1]]]),
+}
+
+DATA_3 = {
+    "image": BATCH_IMAGE,
+    "label": BATCH_LABEL,
+    "pred": np.array([[[[[0, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 1, 0, 0], [0, 1, 0, 1, 0], [0, 0, 0, 0, 0]]]]]),
+}
+
+DATA_4 = {
+    "image": BATCH_IMAGE,
+    "label": BATCH_LABEL,
+    "guidance": np.array([[[[1, 0, 2, 2]], [[-1, -1, -1, -1]]]]),
+    "discrepancy": np.array(
+        [
+            [
+                [[[[0, 0, 0, 0, 0], [0, 0, 0, 1, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0]]]],
+                [[[[0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0]]]],
+            ]
+        ]
+    ),
+    "probability": [1.0],
+}
+
+FIND_SLICE_TEST_CASE_1 = [
+    {"label": "label", "sids": "sids"},
+    DATA_1,
+    [0],
+]
+
+FIND_SLICE_TEST_CASE_2 = [
+    {"label": "label", "sids": "sids"},
+    DATA_2,
+    [0, 1],
+]
+
+CROP_TEST_CASE_1 = [
+    {
+        "keys": ["image", "label"],
+        "source_key": "label",
+        "select_fn": lambda x: x > 0,
+        "channel_indices": None,
+        "margin": 0,
+        "spatial_size": [1, 4, 4],
+    },
+    DATA_1,
+    np.array([[[[1, 2, 1], [2, 3, 2], [1, 2, 1]]]]),
+]
+
+ADD_INITIAL_POINT_TEST_CASE_1 = [
+    {"label": "label", "guidance": "guidance", "sids": "sids"},
+    DATA_1,
+    np.array([[[1, 0, 2, 2]], [[-1, -1, -1, -1]]]),
+]
+
+ADD_GUIDANCE_TEST_CASE_1 = [
+    {"image": "image", "guidance": "guidance", "batched": False},
+    DATA_2,
+    np.array(
+        [
+            [
+                [[1, 2, 3, 2, 1], [1, 1, 3, 2, 1], [0, 0, 0, 0, 0], [1, 1, 1, 2, 1], [0, 2, 2, 2, 1]],
+                [[1, 0, 2, 0, 1], [0, 1, 2, 1, 0], [2, 2, 3, 2, 2], [0, 1, 2, 1, 0], [1, 0, 2, 0, 1]],
+            ],
+            [
+                [
+                    [0.0, 0.26689214, 0.37996644, 0.26689214, 0.0],
+                    [0.26689214, 0.65222847, 0.81548417, 0.65222847, 0.26689214],
+                    [0.37996635, 0.81548399, 1.0, 0.81548399, 0.37996635],
+                    [0.26689214, 0.65222847, 0.81548417, 0.65222847, 0.26689214],
+                    [0.0, 0.26689214, 0.37996644, 0.26689214, 0.0],
+                ],
+                [
+                    [0.0, 0.26689214, 0.37996644, 0.26689214, 0.0],
+                    [0.26689214, 0.65222847, 0.81548417, 0.65222847, 0.26689214],
+                    [0.37996635, 0.81548399, 1.0, 0.81548399, 0.37996635],
+                    [0.26689214, 0.65222847, 0.81548417, 0.65222847, 0.26689214],
+                    [0.0, 0.26689214, 0.37996644, 0.26689214, 0.0],
+                ],
+            ],
+            [
+                [[0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0]],
+                [[0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0]],
+            ],
+        ]
+    ),
+]
+
+FIND_DISCREPANCY_TEST_CASE_1 = [
+    {"label": "label", "pred": "pred", "discrepancy": "discrepancy"},
+    DATA_3,
+    np.array(
+        [
+            [
+                [[[[0, 0, 0, 0, 0], [0, 0, 0, 1, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0]]]],
+                [[[[0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0]]]],
+            ]
+        ]
+    ),
+]
+
+ADD_RANDOM_GUIDANCE_TEST_CASE_1 = [
+    {"guidance": "guidance", "discrepancy": "discrepancy", "probability": "probability", "batched": True},
+    DATA_4,
+    np.array([[[[1, 0, 2, 2], [1, 0, 1, 3]], [[-1, -1, -1, -1], [-1, -1, -1, -1]]]]),
+]
+
+
+class TestFindAllValidSlicesd(unittest.TestCase):
+    @parameterized.expand([FIND_SLICE_TEST_CASE_1, FIND_SLICE_TEST_CASE_2])
+    def test_correct_results(self, arguments, input_data, expected_result):
+        result = FindAllValidSlicesd(**arguments)(input_data)
+        np.testing.assert_allclose(result[arguments["sids"]], expected_result)
+
+
+class TestSpatialCropForegroundd(unittest.TestCase):
+    @parameterized.expand([CROP_TEST_CASE_1])
+    def test_correct_results(self, arguments, input_data, expected_result):
+        result = SpatialCropForegroundd(**arguments)(input_data)
+        np.testing.assert_allclose(result["image"], expected_result)
+
+    @parameterized.expand([CROP_TEST_CASE_1])
+    def test_foreground_position(self, arguments, input_data, _):
+        result = SpatialCropForegroundd(**arguments)(input_data)
+        np.testing.assert_allclose(result["image_meta_dict"]["foreground_start_coord"], np.array([0, 1, 1]))
+        np.testing.assert_allclose(result["image_meta_dict"]["foreground_end_coord"], np.array([1, 4, 4]))
+
+        arguments["start_coord_key"] = "test_start_coord"
+        arguments["end_coord_key"] = "test_end_coord"
+        result = SpatialCropForegroundd(**arguments)(input_data)
+        np.testing.assert_allclose(result["image_meta_dict"]["test_start_coord"], np.array([0, 1, 1]))
+        np.testing.assert_allclose(result["image_meta_dict"]["test_end_coord"], np.array([1, 4, 4]))
+
+
+class TestAddInitialSeedPointd(unittest.TestCase):
+    @parameterized.expand([ADD_INITIAL_POINT_TEST_CASE_1])
+    def test_correct_results(self, arguments, input_data, expected_result):
+        seed = 0
+        add_fn = AddInitialSeedPointd(**arguments)
+        add_fn.set_random_state(seed)
+        result = add_fn(input_data)
+        np.testing.assert_allclose(result[arguments["guidance"]], expected_result)
+
+
+class TestAddGuidanceSignald(unittest.TestCase):
+    @parameterized.expand([ADD_GUIDANCE_TEST_CASE_1])
+    def test_correct_results(self, arguments, input_data, expected_result):
+        result = AddGuidanceSignald(**arguments)(input_data)
+        np.testing.assert_allclose(result["image"], expected_result, rtol=1e-5)
+
+
+class TestFindDiscrepancyRegionsd(unittest.TestCase):
+    @parameterized.expand([FIND_DISCREPANCY_TEST_CASE_1])
+    def test_correct_results(self, arguments, input_data, expected_result):
+        result = FindDiscrepancyRegionsd(**arguments)(input_data)
+        np.testing.assert_allclose(result[arguments["discrepancy"]], expected_result)
+
+
+class TestAddRandomGuidanced(unittest.TestCase):
+    @parameterized.expand([ADD_RANDOM_GUIDANCE_TEST_CASE_1])
+    def test_correct_results(self, arguments, input_data, expected_result):
+        seed = 0
+        add_fn = AddRandomGuidanced(**arguments)
+        add_fn.set_random_state(seed)
+        result = add_fn(input_data)
+        np.testing.assert_allclose(result[arguments["guidance"]], expected_result, rtol=1e-5)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_detect_envelope.py b/tests/test_detect_envelope.py
index 08c699c84f..47b3a66305 100644
--- a/tests/test_detect_envelope.py
+++ b/tests/test_detect_envelope.py
@@ -98,7 +98,7 @@
 
 TEST_CASE_INVALID_DTYPE = [
     {},
-    np.expand_dims(np.array(hann_windowed_sine, dtype=np.complex), 0),  # complex numbers are invalid
+    np.expand_dims(np.array(hann_windowed_sine, dtype=complex), 0),  # complex numbers are invalid
     "__call__",  # method expected to raise exception
 ]
 
diff --git a/tests/test_dvf2ddf.py b/tests/test_dvf2ddf.py
new file mode 100644
index 0000000000..0ee8ba6c30
--- /dev/null
+++ b/tests/test_dvf2ddf.py
@@ -0,0 +1,58 @@
+import unittest
+
+import numpy as np
+import torch
+from parameterized import parameterized
+from torch import nn
+from torch.optim import SGD
+
+from monai.networks.blocks.warp import DVF2DDF
+from monai.utils import set_determinism
+
+TEST_CASES = [
+    [{"spatial_dims": 2, "num_steps": 1}, {"dvf": torch.zeros(1, 2, 2, 2)}, torch.zeros(1, 2, 2, 2)],
+    [
+        {"spatial_dims": 3, "num_steps": 1},
+        {"dvf": torch.ones(1, 3, 2, 2, 2)},
+        torch.tensor([[[1.0000, 0.7500], [0.7500, 0.6250]], [[0.7500, 0.6250], [0.6250, 0.5625]]])
+        .reshape(1, 1, 2, 2, 2)
+        .expand(-1, 3, -1, -1, -1),
+    ],
+    [
+        {"spatial_dims": 3, "num_steps": 2},
+        {"dvf": torch.ones(1, 3, 2, 2, 2)},
+        torch.tensor([[[0.9175, 0.6618], [0.6618, 0.5306]], [[0.6618, 0.5306], [0.5306, 0.4506]]])
+        .reshape(1, 1, 2, 2, 2)
+        .expand(-1, 3, -1, -1, -1),
+    ],
+]
+
+
+class TestDVF2DDF(unittest.TestCase):
+    def setUp(self):
+        set_determinism(0)
+
+    def tearDown(self):
+        set_determinism(None)
+
+    @parameterized.expand(TEST_CASES)
+    def test_value(self, input_param, input_data, expected_val):
+        layer = DVF2DDF(**input_param)
+        result = layer(**input_data)
+        np.testing.assert_allclose(result.cpu().numpy(), expected_val.cpu().numpy(), rtol=1e-4, atol=1e-4)
+
+    def test_gradient(self):
+        network = nn.Conv2d(in_channels=1, out_channels=2, kernel_size=1)
+        dvf2ddf = DVF2DDF(spatial_dims=2, num_steps=1)
+        optimizer = SGD(network.parameters(), lr=0.01)
+        x = torch.ones((1, 1, 5, 5))
+        x = network(x)
+        x = dvf2ddf(x)
+        loss = torch.sum(x)
+        loss.backward()
+        optimizer.step()
+        np.testing.assert_allclose(network.weight.grad.cpu().numpy(), np.array([[[[22.471329]]], [[[22.552576]]]]))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_dynunet.py b/tests/test_dynunet.py
index d72c1fc48a..05e0c17465 100644
--- a/tests/test_dynunet.py
+++ b/tests/test_dynunet.py
@@ -43,6 +43,7 @@
                         "strides": strides,
                         "upsample_kernel_size": strides[1:],
                         "norm_name": "batch",
+                        "deep_supervision": False,
                         "res_block": res_block,
                     },
                     (1, in_channels, in_size, in_size),
@@ -65,6 +66,7 @@
                 "strides": ((1, 2, 1), 2, 2, 1),
                 "upsample_kernel_size": (2, 2, 1),
                 "norm_name": "instance",
+                "deep_supervision": False,
                 "res_block": res_block,
             },
             (1, in_channels, in_size, in_size, in_size),
@@ -77,6 +79,7 @@
     for res_block in [True, False]:
         for deep_supr_num in [1, 2]:
             for strides in [(1, 2, 1, 2, 1), (2, 2, 2, 1), (2, 1, 1, 2, 2)]:
+                scale = strides[0]
                 test_case = [
                     {
                         "spatial_dims": spatial_dims,
@@ -86,18 +89,13 @@
                         "strides": strides,
                         "upsample_kernel_size": strides[1:],
                         "norm_name": "group",
+                        "deep_supervision": True,
                         "deep_supr_num": deep_supr_num,
                         "res_block": res_block,
                     },
                     (1, 1, *[in_size] * spatial_dims),
+                    (1, 1 + deep_supr_num, 2, *[in_size // scale] * spatial_dims),
                 ]
-                scale = 1
-                all_expected_shapes = []
-                for stride in strides[: 1 + deep_supr_num]:
-                    scale *= stride
-                    deep_out_shape = (1, 2, *[in_size // scale] * spatial_dims)
-                    all_expected_shapes.append(deep_out_shape)
-                test_case.append(all_expected_shapes)
                 TEST_CASE_DEEP_SUPERVISION.append(test_case)
 
 
@@ -121,11 +119,8 @@ class TestDynUNetDeepSupervision(unittest.TestCase):
     def test_shape(self, input_param, input_shape, expected_shape):
         net = DynUNet(**input_param).to(device)
         with torch.no_grad():
-            results = [net(torch.randn(input_shape).to(device))] + net.get_feature_maps()
-            self.assertEqual(len(results), len(expected_shape))
-            for idx in range(len(results)):
-                result, sub_expected_shape = results[idx], expected_shape[idx]
-                self.assertEqual(result.shape, sub_expected_shape)
+            results = net(torch.randn(input_shape).to(device))
+            self.assertEqual(results.shape, expected_shape)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_ensemble_evaluator.py b/tests/test_ensemble_evaluator.py
index fdb9695476..9cc977d876 100644
--- a/tests/test_ensemble_evaluator.py
+++ b/tests/test_ensemble_evaluator.py
@@ -55,7 +55,7 @@ def forward(self, x):
         def run_post_transform(engine):
             for i in range(5):
                 expected_value = engine.state.iteration + i
-                torch.testing.assert_allclose(engine.state.output[f"pred{i}"], expected_value)
+                torch.testing.assert_allclose(engine.state.output[f"pred{i}"], torch.tensor([[expected_value]]))
 
         val_engine.run()
 
diff --git a/tests/test_flip.py b/tests/test_flip.py
index 7a2af02585..fe169c4da8 100644
--- a/tests/test_flip.py
+++ b/tests/test_flip.py
@@ -19,7 +19,7 @@
 
 INVALID_CASES = [("wrong_axis", ["s", 1], TypeError), ("not_numbers", "s", TypeError)]
 
-VALID_CASES = [("no_axis", None), ("one_axis", 1), ("many_axis", [0, 1])]
+VALID_CASES = [("no_axis", None), ("one_axis", 1), ("many_axis", [0, 1]), ("negative_axis", [0, -1])]
 
 
 class TestFlip(NumpyImageTestCase2D):
diff --git a/tests/test_handler_checkpoint_loader.py b/tests/test_handler_checkpoint_loader.py
index d299b65e9b..8b0f752ff4 100644
--- a/tests/test_handler_checkpoint_loader.py
+++ b/tests/test_handler_checkpoint_loader.py
@@ -40,7 +40,7 @@ def test_one_save_one_load(self):
             engine = Engine(lambda e, b: None)
             CheckpointLoader(load_path=path, load_dict={"net": net2}).attach(engine)
             engine.run([0] * 8, max_epochs=1)
-            torch.testing.assert_allclose(net2.state_dict()["weight"], 0.1)
+            torch.testing.assert_allclose(net2.state_dict()["weight"], torch.tensor([0.1]))
 
     def test_two_save_one_load(self):
         logging.basicConfig(stream=sys.stdout, level=logging.INFO)
@@ -62,7 +62,7 @@ def test_two_save_one_load(self):
             engine = Engine(lambda e, b: None)
             CheckpointLoader(load_path=path, load_dict={"net": net2}).attach(engine)
             engine.run([0] * 8, max_epochs=1)
-            torch.testing.assert_allclose(net2.state_dict()["weight"], 0.1)
+            torch.testing.assert_allclose(net2.state_dict()["weight"], torch.tensor([0.1]))
 
     def test_save_single_device_load_multi_devices(self):
         logging.basicConfig(stream=sys.stdout, level=logging.INFO)
@@ -83,7 +83,7 @@ def test_save_single_device_load_multi_devices(self):
             engine = Engine(lambda e, b: None)
             CheckpointLoader(load_path=path, load_dict={"net": net2}).attach(engine)
             engine.run([0] * 8, max_epochs=1)
-            torch.testing.assert_allclose(net2.state_dict()["module.weight"].cpu(), 0.1)
+            torch.testing.assert_allclose(net2.state_dict()["module.weight"].cpu(), torch.tensor([0.1]))
 
 
 if __name__ == "__main__":
diff --git a/tests/test_handler_checkpoint_saver.py b/tests/test_handler_checkpoint_saver.py
index 0ae8be1e73..5c2b750a57 100644
--- a/tests/test_handler_checkpoint_saver.py
+++ b/tests/test_handler_checkpoint_saver.py
@@ -20,9 +20,9 @@
 from ignite.engine import Engine
 from parameterized import parameterized
 
-from monai.handlers import CheckpointSaver
+from monai.handlers import CheckpointLoader, CheckpointSaver
 
-TEST_CASE_1 = [True, None, False, None, 1, None, True, 0, None, ["test_checkpoint_final_iteration=40.pt"]]
+TEST_CASE_1 = [True, None, False, None, 1, None, False, True, 0, None, ["test_checkpoint_final_iteration=40.pt"]]
 
 TEST_CASE_2 = [
     False,
@@ -31,6 +31,7 @@
     "val_loss",
     2,
     None,
+    False,
     True,
     0,
     None,
@@ -44,6 +45,7 @@
     None,
     1,
     None,
+    False,
     True,
     2,
     2,
@@ -58,16 +60,17 @@
     1,
     None,
     False,
+    False,
     10,
     2,
     ["test_checkpoint_iteration=30.pt", "test_checkpoint_iteration=40.pt"],
 ]
 
-TEST_CASE_5 = [True, None, False, None, 1, None, True, 0, None, ["test_checkpoint_final_iteration=40.pt"], True]
+TEST_CASE_5 = [True, None, False, None, 1, None, False, True, 0, None, ["test_checkpoint_final_iteration=40.pt"], True]
 
-TEST_CASE_6 = [True, "final_model.pt", False, None, 1, None, True, 0, None, ["final_model.pt"]]
+TEST_CASE_6 = [True, "final_model.pt", False, None, 1, None, False, True, 0, None, ["final_model.pt"]]
 
-TEST_CASE_7 = [False, None, True, "val_loss", 1, "model.pt", True, 0, None, ["model.pt"]]
+TEST_CASE_7 = [False, None, True, "val_loss", 1, "model.pt", False, True, 0, None, ["model.pt"]]
 
 
 class TestHandlerCheckpointSaver(unittest.TestCase):
@@ -80,6 +83,7 @@ def test_file(
         key_metric_name,
         key_metric_n_saved,
         key_metric_filename,
+        key_metric_save_state,
         epoch_level,
         save_interval,
         n_saved,
@@ -112,6 +116,7 @@ def _train_func(engine, batch):
                 key_metric_name,
                 key_metric_n_saved,
                 key_metric_filename,
+                key_metric_save_state,
                 epoch_level,
                 save_interval,
                 n_saved,
@@ -141,6 +146,45 @@ def _train_func(engine, batch):
                 engine.run(range(3), max_epochs=2)
             self.assertTrue(os.path.exists(os.path.join(tempdir, "net_final_iteration=1.pt")))
 
+    def test_load_state_dict(self):
+        logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+        net = torch.nn.PReLU()
+
+        # set up engine
+        def _train_func(engine, batch):
+            engine.state.metrics["val_loss"] = engine.state.iteration
+
+        engine = Engine(_train_func)
+
+        # set up testing handler
+        with tempfile.TemporaryDirectory() as tempdir:
+            engine = Engine(_train_func)
+            CheckpointSaver(
+                save_dir=tempdir,
+                save_dict={"net": net},
+                save_key_metric=True,
+                key_metric_name="val_loss",
+                key_metric_n_saved=2,
+                key_metric_save_state=True,
+            ).attach(engine)
+            engine.run(range(3), max_epochs=2)
+
+            saver = CheckpointSaver(
+                save_dir=tempdir,
+                save_dict={"net": net},
+                save_key_metric=True,
+                key_metric_name="val_loss",
+                key_metric_n_saved=2,
+            )
+            engine = Engine(_train_func)
+            CheckpointLoader(os.path.join(tempdir, "net_key_metric=6.pt"), {"checkpointer": saver}).attach(engine)
+            engine.run(range(1), max_epochs=1)
+
+            resumed = saver._key_metric_checkpoint._saved
+            for i in range(2):
+                self.assertEqual(resumed[i].priority, 3 * (i + 1))
+                self.assertEqual(resumed[i].filename, f"net_key_metric={3 * (i + 1)}.pt")
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_handler_classification_saver_dist.py b/tests/test_handler_classification_saver_dist.py
new file mode 100644
index 0000000000..a33cba923a
--- /dev/null
+++ b/tests/test_handler_classification_saver_dist.py
@@ -0,0 +1,65 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import csv
+import os
+import tempfile
+import unittest
+
+import numpy as np
+import torch
+import torch.distributed as dist
+from ignite.engine import Engine
+
+from monai.handlers import ClassificationSaver
+from tests.utils import DistCall, DistTestCase, SkipIfBeforePyTorchVersion
+
+
+@SkipIfBeforePyTorchVersion((1, 7))
+class DistributedHandlerClassificationSaver(DistTestCase):
+    @DistCall(nnodes=1, nproc_per_node=2)
+    def test_saved_content(self):
+        with tempfile.TemporaryDirectory() as tempdir:
+            rank = dist.get_rank()
+
+            # set up engine
+            def _train_func(engine, batch):
+                return torch.zeros(8 + rank * 2)
+
+            engine = Engine(_train_func)
+
+            # set up testing handler
+            saver = ClassificationSaver(output_dir=tempdir, filename="predictions.csv", save_rank=1)
+            saver.attach(engine)
+
+            # rank 0 has 8 images, rank 1 has 10 images
+            data = [
+                {
+                    "filename_or_obj": ["testfile" + str(i) for i in range(8 * rank, (8 + rank) * (rank + 1))],
+                    "data_shape": [(1, 1) for _ in range(8 * rank, (8 + rank) * (rank + 1))],
+                }
+            ]
+            engine.run(data, max_epochs=1)
+            filepath = os.path.join(tempdir, "predictions.csv")
+            if rank == 1:
+                self.assertTrue(os.path.exists(filepath))
+                with open(filepath, "r") as f:
+                    reader = csv.reader(f)
+                    i = 0
+                    for row in reader:
+                        self.assertEqual(row[0], "testfile" + str(i))
+                        self.assertEqual(np.array(row[1:]).astype(np.float32), 0.0)
+                        i += 1
+                    self.assertEqual(i, 18)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_handler_rocauc_dist.py b/tests/test_handler_rocauc_dist.py
index 825b172064..c5cf44162c 100644
--- a/tests/test_handler_rocauc_dist.py
+++ b/tests/test_handler_rocauc_dist.py
@@ -31,12 +31,12 @@ def test_compute(self):
             auc_metric.update([y_pred, y])
 
         if dist.get_rank() == 1:
-            y_pred = torch.tensor([[0.2, 0.1], [0.1, 0.5]], device=device)
-            y = torch.tensor([[0], [1]], device=device)
+            y_pred = torch.tensor([[0.2, 0.1], [0.1, 0.5], [0.3, 0.4]], device=device)
+            y = torch.tensor([[0], [1], [1]], device=device)
             auc_metric.update([y_pred, y])
 
         result = auc_metric.compute()
-        np.testing.assert_allclose(0.75, result)
+        np.testing.assert_allclose(0.66667, result, rtol=1e-4)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_handler_tb_image.py b/tests/test_handler_tb_image.py
index ed3ba8a32d..f946fb6060 100644
--- a/tests/test_handler_tb_image.py
+++ b/tests/test_handler_tb_image.py
@@ -40,6 +40,7 @@ def _train_func(engine, batch):
 
             data = zip(np.random.normal(size=(10, 4, *shape)), np.random.normal(size=(10, 4, *shape)))
             engine.run(data, epoch_length=10, max_epochs=1)
+            stats_handler.close()
 
             self.assertTrue(len(glob.glob(tempdir)) > 0)
 
diff --git a/tests/test_handler_tb_stats.py b/tests/test_handler_tb_stats.py
index 2d7d18d1f6..0d8654cb09 100644
--- a/tests/test_handler_tb_stats.py
+++ b/tests/test_handler_tb_stats.py
@@ -39,6 +39,7 @@ def _update_metric(engine):
             stats_handler = TensorBoardStatsHandler(log_dir=tempdir)
             stats_handler.attach(engine)
             engine.run(range(3), max_epochs=2)
+            stats_handler.close()
             # check logging output
             self.assertTrue(len(glob.glob(tempdir)) > 0)
 
@@ -64,6 +65,7 @@ def _update_metric(engine):
             )
             stats_handler.attach(engine)
             engine.run(range(3), max_epochs=2)
+            writer.close()
             # check logging output
             self.assertTrue(len(glob.glob(tempdir)) > 0)
 
diff --git a/tests/test_integration_workflows.py b/tests/test_integration_workflows.py
index aa4ccbb76d..db7580bf86 100644
--- a/tests/test_integration_workflows.py
+++ b/tests/test_integration_workflows.py
@@ -22,6 +22,7 @@
 import numpy as np
 import torch
 from ignite.metrics import Accuracy
+from torch.utils.tensorboard import SummaryWriter
 
 import monai
 from monai.data import create_test_image_3d
@@ -105,6 +106,7 @@ def run_training_test(root_dir, device="cuda:0", amp=False, num_workers=4):
     loss = monai.losses.DiceLoss(sigmoid=True)
     opt = torch.optim.Adam(net.parameters(), 1e-3)
     lr_scheduler = torch.optim.lr_scheduler.StepLR(opt, step_size=2, gamma=0.1)
+    summary_writer = SummaryWriter(log_dir=root_dir)
 
     val_post_transforms = Compose(
         [
@@ -123,7 +125,7 @@ def _forward_completed(self, engine):
 
     val_handlers = [
         StatsHandler(output_transform=lambda x: None),
-        TensorBoardStatsHandler(log_dir=root_dir, output_transform=lambda x: None),
+        TensorBoardStatsHandler(summary_writer=summary_writer, output_transform=lambda x: None),
         TensorBoardImageHandler(
             log_dir=root_dir, batch_transform=lambda x: (x["image"], x["label"]), output_transform=lambda x: x["pred"]
         ),
@@ -176,7 +178,9 @@ def _optimizer_completed(self, engine):
         LrScheduleHandler(lr_scheduler=lr_scheduler, print_lr=True),
         ValidationHandler(validator=evaluator, interval=2, epoch_level=True),
         StatsHandler(tag_name="train_loss", output_transform=lambda x: x["loss"]),
-        TensorBoardStatsHandler(log_dir=root_dir, tag_name="train_loss", output_transform=lambda x: x["loss"]),
+        TensorBoardStatsHandler(
+            summary_writer=summary_writer, tag_name="train_loss", output_transform=lambda x: x["loss"]
+        ),
         CheckpointSaver(save_dir=root_dir, save_dict={"net": net, "opt": opt}, save_interval=2, epoch_level=True),
         _TestTrainIterEvents(),
     ]
diff --git a/tests/test_multi_scale.py b/tests/test_multi_scale.py
new file mode 100644
index 0000000000..722ae7cfce
--- /dev/null
+++ b/tests/test_multi_scale.py
@@ -0,0 +1,60 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import unittest
+
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.losses import DiceLoss
+from monai.losses.multi_scale import MultiScaleLoss
+
+dice_loss = DiceLoss(include_background=True, sigmoid=True, smooth_nr=1e-5, smooth_dr=1e-5)
+
+TEST_CASES = [
+    [
+        {"loss": dice_loss, "scales": None, "kernel": "gaussian"},
+        {"y_pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]), "y_true": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]])},
+        0.307576,
+    ],
+    [
+        {"loss": dice_loss, "scales": [0, 1], "kernel": "gaussian"},
+        {"y_pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]), "y_true": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]])},
+        0.463116,
+    ],
+    [
+        {"loss": dice_loss, "scales": [0, 1, 2], "kernel": "cauchy"},
+        {
+            "y_pred": torch.tensor([[[[[1.0, -1.0], [-1.0, 1.0]]]]]),
+            "y_true": torch.tensor([[[[[1.0, 0.0], [1.0, 1.0]]]]]),
+        },
+        0.715228,
+    ],
+]
+
+
+class TestMultiScale(unittest.TestCase):
+    @parameterized.expand(TEST_CASES)
+    def test_shape(self, input_param, input_data, expected_val):
+        result = MultiScaleLoss(**input_param).forward(**input_data)
+        np.testing.assert_allclose(result.detach().cpu().numpy(), expected_val, rtol=1e-5)
+
+    def test_ill_opts(self):
+        with self.assertRaisesRegex(ValueError, ""):
+            MultiScaleLoss(loss=dice_loss, kernel="none")
+        with self.assertRaisesRegex(ValueError, ""):
+            MultiScaleLoss(loss=dice_loss, scales=[-1])(torch.ones((1, 1, 3)), torch.ones((1, 1, 3)))
+        with self.assertRaisesRegex(ValueError, ""):
+            MultiScaleLoss(loss=dice_loss, scales=[-1], reduction="none")(torch.ones((1, 1, 3)), torch.ones((1, 1, 3)))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_patch_dataset.py b/tests/test_patch_dataset.py
index 59174123ca..3dadbe3d92 100644
--- a/tests/test_patch_dataset.py
+++ b/tests/test_patch_dataset.py
@@ -42,7 +42,7 @@ def test_shape(self):
     def test_loading_array(self):
         set_determinism(seed=1234)
         # image dataset
-        images = [np.arange(16, dtype=np.float).reshape(1, 4, 4), np.arange(16, dtype=np.float).reshape(1, 4, 4)]
+        images = [np.arange(16, dtype=float).reshape(1, 4, 4), np.arange(16, dtype=float).reshape(1, 4, 4)]
         # image patch sampler
         n_samples = 8
         sampler = RandSpatialCropSamples(roi_size=(3, 3), num_samples=n_samples, random_center=True, random_size=False)
diff --git a/tests/test_phl_cpu.py b/tests/test_phl_cpu.py
index f0e62cbddb..ec6d58824d 100644
--- a/tests/test_phl_cpu.py
+++ b/tests/test_phl_cpu.py
@@ -16,7 +16,7 @@
 from parameterized import parameterized
 
 from monai.networks.layers.filtering import PHLFilter
-from tests.utils import skip_if_no_cpp_extention
+from tests.utils import skip_if_no_cpp_extension
 
 TEST_CASES = [
     [
@@ -238,7 +238,7 @@
 ]
 
 
-@skip_if_no_cpp_extention
+@skip_if_no_cpp_extension
 class PHLFilterTestCaseCpu(unittest.TestCase):
     @parameterized.expand(TEST_CASES)
     def test_cpu(self, test_case_description, sigmas, input, features, expected):
diff --git a/tests/test_phl_cuda.py b/tests/test_phl_cuda.py
index 8b89efce1a..d7538f14fa 100644
--- a/tests/test_phl_cuda.py
+++ b/tests/test_phl_cuda.py
@@ -16,7 +16,7 @@
 from parameterized import parameterized
 
 from monai.networks.layers.filtering import PHLFilter
-from tests.utils import skip_if_no_cpp_extention, skip_if_no_cuda
+from tests.utils import skip_if_no_cpp_extension, skip_if_no_cuda
 
 TEST_CASES = [
     [
@@ -146,7 +146,7 @@
 
 
 @skip_if_no_cuda
-@skip_if_no_cpp_extention
+@skip_if_no_cpp_extension
 class PHLFilterTestCaseCuda(unittest.TestCase):
     @parameterized.expand(TEST_CASES)
     def test_cuda(self, test_case_description, sigmas, input, features, expected):
diff --git a/tests/test_rand_affine.py b/tests/test_rand_affine.py
index 72fa772d96..68126f5c8e 100644
--- a/tests/test_rand_affine.py
+++ b/tests/test_rand_affine.py
@@ -74,8 +74,8 @@ def test_rand_affine(self, input_param, input_data, expected_val):
         g = RandAffine(**input_param)
         g.set_random_state(123)
         result = g(**input_data)
-        self.assertEqual(torch.is_tensor(result), torch.is_tensor(expected_val))
-        if torch.is_tensor(result):
+        self.assertEqual(isinstance(result, torch.Tensor), isinstance(expected_val, torch.Tensor))
+        if isinstance(result, torch.Tensor):
             np.testing.assert_allclose(result.cpu().numpy(), expected_val.cpu().numpy(), rtol=1e-4, atol=1e-4)
         else:
             np.testing.assert_allclose(result, expected_val, rtol=1e-4, atol=1e-4)
diff --git a/tests/test_rand_affine_grid.py b/tests/test_rand_affine_grid.py
index c3fe078afd..605d0a30ba 100644
--- a/tests/test_rand_affine_grid.py
+++ b/tests/test_rand_affine_grid.py
@@ -187,8 +187,8 @@ def test_rand_affine_grid(self, input_param, input_data, expected_val):
         g = RandAffineGrid(**input_param)
         g.set_random_state(123)
         result = g(**input_data)
-        self.assertEqual(torch.is_tensor(result), torch.is_tensor(expected_val))
-        if torch.is_tensor(result):
+        self.assertEqual(isinstance(result, torch.Tensor), isinstance(expected_val, torch.Tensor))
+        if isinstance(result, torch.Tensor):
             np.testing.assert_allclose(result.cpu().numpy(), expected_val.cpu().numpy(), rtol=1e-4, atol=1e-4)
         else:
             np.testing.assert_allclose(result, expected_val, rtol=1e-4, atol=1e-4)
diff --git a/tests/test_rand_affined.py b/tests/test_rand_affined.py
index 51bb59cd37..54d71ad8f7 100644
--- a/tests/test_rand_affined.py
+++ b/tests/test_rand_affined.py
@@ -146,8 +146,8 @@ def test_rand_affined(self, input_param, input_data, expected_val):
         for key in res:
             result = res[key]
             expected = expected_val[key] if isinstance(expected_val, dict) else expected_val
-            self.assertEqual(torch.is_tensor(result), torch.is_tensor(expected))
-            if torch.is_tensor(result):
+            self.assertEqual(isinstance(result, torch.Tensor), isinstance(expected, torch.Tensor))
+            if isinstance(result, torch.Tensor):
                 np.testing.assert_allclose(result.cpu().numpy(), expected.cpu().numpy(), rtol=1e-4, atol=1e-4)
             else:
                 np.testing.assert_allclose(result, expected, rtol=1e-4, atol=1e-4)
diff --git a/tests/test_rand_deform_grid.py b/tests/test_rand_deform_grid.py
index 0b969f8f4b..7c12c263d2 100644
--- a/tests/test_rand_deform_grid.py
+++ b/tests/test_rand_deform_grid.py
@@ -129,8 +129,8 @@ def test_rand_deform_grid(self, input_param, input_data, expected_val):
         g = RandDeformGrid(**input_param)
         g.set_random_state(123)
         result = g(**input_data)
-        self.assertEqual(torch.is_tensor(result), torch.is_tensor(expected_val))
-        if torch.is_tensor(result):
+        self.assertEqual(isinstance(result, torch.Tensor), isinstance(expected_val, torch.Tensor))
+        if isinstance(result, torch.Tensor):
             np.testing.assert_allclose(result.cpu().numpy(), expected_val.cpu().numpy(), rtol=1e-4, atol=1e-4)
         else:
             np.testing.assert_allclose(result, expected_val, rtol=1e-4, atol=1e-4)
diff --git a/tests/test_rand_elastic_2d.py b/tests/test_rand_elastic_2d.py
index c9db225742..aa408f0fdc 100644
--- a/tests/test_rand_elastic_2d.py
+++ b/tests/test_rand_elastic_2d.py
@@ -95,8 +95,8 @@ def test_rand_2d_elastic(self, input_param, input_data, expected_val):
         g = Rand2DElastic(**input_param)
         g.set_random_state(123)
         result = g(**input_data)
-        self.assertEqual(torch.is_tensor(result), torch.is_tensor(expected_val))
-        if torch.is_tensor(result):
+        self.assertEqual(isinstance(result, torch.Tensor), isinstance(expected_val, torch.Tensor))
+        if isinstance(result, torch.Tensor):
             np.testing.assert_allclose(result.cpu().numpy(), expected_val.cpu().numpy(), rtol=1e-4, atol=1e-4)
         else:
             np.testing.assert_allclose(result, expected_val, rtol=1e-4, atol=1e-4)
diff --git a/tests/test_rand_elastic_3d.py b/tests/test_rand_elastic_3d.py
index f2b1669a46..8cd74c6be7 100644
--- a/tests/test_rand_elastic_3d.py
+++ b/tests/test_rand_elastic_3d.py
@@ -74,8 +74,8 @@ def test_rand_3d_elastic(self, input_param, input_data, expected_val):
         g = Rand3DElastic(**input_param)
         g.set_random_state(123)
         result = g(**input_data)
-        self.assertEqual(torch.is_tensor(result), torch.is_tensor(expected_val))
-        if torch.is_tensor(result):
+        self.assertEqual(isinstance(result, torch.Tensor), isinstance(expected_val, torch.Tensor))
+        if isinstance(result, torch.Tensor):
             np.testing.assert_allclose(result.cpu().numpy(), expected_val.cpu().numpy(), rtol=1e-4, atol=1e-4)
         else:
             np.testing.assert_allclose(result, expected_val, rtol=1e-4, atol=1e-4)
diff --git a/tests/test_rand_elasticd_2d.py b/tests/test_rand_elasticd_2d.py
index 054a0c2150..f8eb026088 100644
--- a/tests/test_rand_elasticd_2d.py
+++ b/tests/test_rand_elasticd_2d.py
@@ -144,8 +144,8 @@ def test_rand_2d_elasticd(self, input_param, input_data, expected_val):
         for key in res:
             result = res[key]
             expected = expected_val[key] if isinstance(expected_val, dict) else expected_val
-            self.assertEqual(torch.is_tensor(result), torch.is_tensor(expected))
-            if torch.is_tensor(result):
+            self.assertEqual(isinstance(result, torch.Tensor), isinstance(expected, torch.Tensor))
+            if isinstance(result, torch.Tensor):
                 np.testing.assert_allclose(result.cpu().numpy(), expected.cpu().numpy(), rtol=1e-4, atol=1e-4)
             else:
                 np.testing.assert_allclose(result, expected, rtol=1e-4, atol=1e-4)
diff --git a/tests/test_rand_elasticd_3d.py b/tests/test_rand_elasticd_3d.py
index 97df8a43e3..47ab814882 100644
--- a/tests/test_rand_elasticd_3d.py
+++ b/tests/test_rand_elasticd_3d.py
@@ -115,8 +115,8 @@ def test_rand_3d_elasticd(self, input_param, input_data, expected_val):
         for key in res:
             result = res[key]
             expected = expected_val[key] if isinstance(expected_val, dict) else expected_val
-            self.assertEqual(torch.is_tensor(result), torch.is_tensor(expected))
-            if torch.is_tensor(result):
+            self.assertEqual(isinstance(result, torch.Tensor), isinstance(expected, torch.Tensor))
+            if isinstance(result, torch.Tensor):
                 np.testing.assert_allclose(result.cpu().numpy(), expected.cpu().numpy(), rtol=1e-4, atol=1e-4)
             else:
                 np.testing.assert_allclose(result, expected, rtol=1e-4, atol=1e-4)
diff --git a/tests/test_rand_gaussian_noised.py b/tests/test_rand_gaussian_noised.py
index 63da23c8dc..442a85ca77 100644
--- a/tests/test_rand_gaussian_noised.py
+++ b/tests/test_rand_gaussian_noised.py
@@ -17,38 +17,36 @@
 from monai.transforms import RandGaussianNoised
 from tests.utils import NumpyImageTestCase2D, TorchImageTestCase2D
 
-TEST_CASE_0 = ["test_zero_mean", ["img"], 0, 0.1]
-TEST_CASE_1 = ["test_non_zero_mean", ["img"], 1, 0.5]
+TEST_CASE_0 = ["test_zero_mean", ["img1", "img2"], 0, 0.1]
+TEST_CASE_1 = ["test_non_zero_mean", ["img1", "img2"], 1, 0.5]
 TEST_CASES = [TEST_CASE_0, TEST_CASE_1]
 
 seed = 0
 
-# Test with numpy
+
+def test_numpy_or_torch(keys, mean, std, imt):
+    gaussian_fn = RandGaussianNoised(keys=keys, prob=1.0, mean=mean, std=std)
+    gaussian_fn.set_random_state(seed)
+    noised = gaussian_fn({k: imt for k in keys})
+    np.random.seed(seed)
+    np.random.random()
+    for k in keys:
+        expected = imt + np.random.normal(mean, np.random.uniform(0, std), size=imt.shape)
+        np.testing.assert_allclose(expected, noised[k], atol=1e-5, rtol=1e-5)
 
 
+# Test with numpy
 class TestRandGaussianNoisedNumpy(NumpyImageTestCase2D):
     @parameterized.expand(TEST_CASES)
     def test_correct_results(self, _, keys, mean, std):
-        gaussian_fn = RandGaussianNoised(keys=keys, prob=1.0, mean=mean, std=std)
-        gaussian_fn.set_random_state(seed)
-        noised = gaussian_fn({"img": self.imt})
-        np.random.seed(seed)
-        np.random.random()
-        expected = self.imt + np.random.normal(mean, np.random.uniform(0, std), size=self.imt.shape)
-        np.testing.assert_allclose(expected, noised["img"], atol=1e-5, rtol=1e-5)
+        test_numpy_or_torch(keys, mean, std, self.imt)
 
 
 # Test with torch
 class TestRandGaussianNoisedTorch(TorchImageTestCase2D):
     @parameterized.expand(TEST_CASES)
     def test_correct_results(self, _, keys, mean, std):
-        gaussian_fn = RandGaussianNoised(keys=keys, prob=1.0, mean=mean, std=std)
-        gaussian_fn.set_random_state(seed)
-        noised = gaussian_fn({"img": self.imt})
-        np.random.seed(seed)
-        np.random.random()
-        expected = self.imt + np.random.normal(mean, np.random.uniform(0, std), size=self.imt.shape)
-        np.testing.assert_allclose(expected, noised["img"], atol=1e-5, rtol=1e-5)
+        test_numpy_or_torch(keys, mean, std, self.imt)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_rand_lambdad.py b/tests/test_rand_lambdad.py
new file mode 100644
index 0000000000..359da8857a
--- /dev/null
+++ b/tests/test_rand_lambdad.py
@@ -0,0 +1,48 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+
+from monai.transforms import Randomizable
+from monai.transforms.utility.dictionary import RandLambdad
+
+
+class RandTest(Randomizable):
+    """
+    randomisable transform for testing.
+    """
+
+    def randomize(self, data=None):
+        self._a = self.R.random()
+
+    def __call__(self, data):
+        self.randomize()
+        return data + self._a
+
+
+class TestRandLambdad(unittest.TestCase):
+    def test_rand_lambdad_identity(self):
+        img = np.zeros((10, 10))
+        data = {"img": img, "prop": 1.0}
+
+        test_func = RandTest()
+        test_func.set_random_state(seed=134)
+        expected = {"img": test_func(data["img"]), "prop": 1.0}
+        test_func.set_random_state(seed=134)
+        ret = RandLambdad(keys=["img", "prop"], func=test_func, overwrite=[True, False])(data)
+        np.testing.assert_allclose(expected["img"], ret["img"])
+        np.testing.assert_allclose(expected["prop"], ret["prop"])
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_resampler.py b/tests/test_resampler.py
index a4536967fa..2be94acebd 100644
--- a/tests/test_resampler.py
+++ b/tests/test_resampler.py
@@ -75,8 +75,8 @@ class TestResample(unittest.TestCase):
     def test_resample(self, input_param, input_data, expected_val):
         g = Resample(**input_param)
         result = g(**input_data)
-        self.assertEqual(torch.is_tensor(result), torch.is_tensor(expected_val))
-        if torch.is_tensor(result):
+        self.assertEqual(isinstance(result, torch.Tensor), isinstance(expected_val, torch.Tensor))
+        if isinstance(result, torch.Tensor):
             np.testing.assert_allclose(result.cpu().numpy(), expected_val.cpu().numpy(), rtol=1e-4, atol=1e-4)
         else:
             np.testing.assert_allclose(result, expected_val, rtol=1e-4, atol=1e-4)
diff --git a/tests/test_rotate90.py b/tests/test_rotate90.py
index a8b4e3f57c..4ab39d5cf6 100644
--- a/tests/test_rotate90.py
+++ b/tests/test_rotate90.py
@@ -37,11 +37,11 @@ def test_k(self):
         self.assertTrue(np.allclose(rotated, expected))
 
     def test_spatial_axes(self):
-        rotate = Rotate90(spatial_axes=(0, 1))
+        rotate = Rotate90(spatial_axes=(0, -1))
         rotated = rotate(self.imt[0])
         expected = []
         for channel in self.imt[0]:
-            expected.append(np.rot90(channel, 1, (0, 1)))
+            expected.append(np.rot90(channel, 1, (0, -1)))
         expected = np.stack(expected)
         self.assertTrue(np.allclose(rotated, expected))
 
diff --git a/tests/test_save_image.py b/tests/test_save_image.py
new file mode 100644
index 0000000000..141960e09b
--- /dev/null
+++ b/tests/test_save_image.py
@@ -0,0 +1,112 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import tempfile
+import unittest
+
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.transforms import SaveImage
+
+TEST_CASE_0 = [
+    torch.randint(0, 255, (8, 1, 2, 3, 4)),
+    {"filename_or_obj": ["testfile" + str(i) + ".nii.gz" for i in range(8)]},
+    ".nii.gz",
+    False,
+    True,
+]
+
+TEST_CASE_1 = [
+    torch.randint(0, 255, (8, 1, 2, 3), dtype=torch.uint8),
+    {"filename_or_obj": ["testfile" + str(i) + ".png" for i in range(8)]},
+    ".png",
+    False,
+    True,
+]
+
+TEST_CASE_2 = [
+    np.random.randint(0, 255, (8, 1, 2, 3, 4)),
+    {"filename_or_obj": ["testfile" + str(i) + ".nii.gz" for i in range(8)]},
+    ".nii.gz",
+    False,
+    True,
+]
+
+TEST_CASE_3 = [
+    torch.randint(0, 255, (8, 1, 2, 2)),
+    {
+        "filename_or_obj": ["testfile" + str(i) + ".nii.gz" for i in range(8)],
+        "spatial_shape": [(28, 28)] * 8,
+        "affine": [np.diag(np.ones(4)) * 5] * 8,
+        "original_affine": [np.diag(np.ones(4)) * 1.0] * 8,
+    },
+    ".nii.gz",
+    True,
+    True,
+]
+
+TEST_CASE_4 = [
+    torch.randint(0, 255, (8, 1, 2, 3), dtype=torch.uint8),
+    {
+        "filename_or_obj": ["testfile" + str(i) + ".png" for i in range(8)],
+        "spatial_shape": [(28, 28)] * 8,
+    },
+    ".png",
+    True,
+    True,
+]
+
+TEST_CASE_5 = [
+    torch.randint(0, 255, (1, 2, 3, 4)),
+    {"filename_or_obj": "testfile0.nii.gz"},
+    ".nii.gz",
+    False,
+    False,
+]
+
+TEST_CASE_6 = [
+    torch.randint(0, 255, (1, 2, 3, 4)),
+    None,
+    ".nii.gz",
+    False,
+    False,
+]
+
+
+class TestSaveImage(unittest.TestCase):
+    @parameterized.expand([TEST_CASE_0, TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6])
+    def test_saved_content(self, test_data, meta_data, output_ext, resample, save_batch):
+        with tempfile.TemporaryDirectory() as tempdir:
+            trans = SaveImage(
+                output_dir=tempdir,
+                output_ext=output_ext,
+                resample=resample,
+                save_batch=save_batch,
+            )
+            trans(test_data, meta_data)
+
+            if save_batch:
+                for i in range(8):
+                    filepath = os.path.join("testfile" + str(i), "testfile" + str(i) + "_trans" + output_ext)
+                    self.assertTrue(os.path.exists(os.path.join(tempdir, filepath)))
+            else:
+                if meta_data is not None:
+                    filepath = os.path.join("testfile0", "testfile0" + "_trans" + output_ext)
+                else:
+                    filepath = os.path.join("0", "0" + "_trans" + output_ext)
+                self.assertTrue(os.path.exists(os.path.join(tempdir, filepath)))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_save_imaged.py b/tests/test_save_imaged.py
new file mode 100644
index 0000000000..a6ebfe0d8d
--- /dev/null
+++ b/tests/test_save_imaged.py
@@ -0,0 +1,114 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import tempfile
+import unittest
+
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.transforms import SaveImaged
+
+TEST_CASE_0 = [
+    {
+        "img": torch.randint(0, 255, (8, 1, 2, 3, 4)),
+        "img_meta_dict": {"filename_or_obj": ["testfile" + str(i) + ".nii.gz" for i in range(8)]},
+    },
+    ".nii.gz",
+    False,
+    True,
+]
+
+TEST_CASE_1 = [
+    {
+        "img": torch.randint(0, 255, (8, 1, 2, 3), dtype=torch.uint8),
+        "img_meta_dict": {"filename_or_obj": ["testfile" + str(i) + ".png" for i in range(8)]},
+    },
+    ".png",
+    False,
+    True,
+]
+
+TEST_CASE_2 = [
+    {
+        "img": np.random.randint(0, 255, (8, 1, 2, 3, 4)),
+        "img_meta_dict": {"filename_or_obj": ["testfile" + str(i) + ".nii.gz" for i in range(8)]},
+    },
+    ".nii.gz",
+    False,
+    True,
+]
+
+TEST_CASE_3 = [
+    {
+        "img": torch.randint(0, 255, (8, 1, 2, 2)),
+        "img_meta_dict": {
+            "filename_or_obj": ["testfile" + str(i) + ".nii.gz" for i in range(8)],
+            "spatial_shape": [(28, 28)] * 8,
+            "affine": [np.diag(np.ones(4)) * 5] * 8,
+            "original_affine": [np.diag(np.ones(4)) * 1.0] * 8,
+        },
+    },
+    ".nii.gz",
+    True,
+    True,
+]
+
+TEST_CASE_4 = [
+    {
+        "img": torch.randint(0, 255, (8, 1, 2, 3), dtype=torch.uint8),
+        "img_meta_dict": {
+            "filename_or_obj": ["testfile" + str(i) + ".png" for i in range(8)],
+            "spatial_shape": [(28, 28)] * 8,
+        },
+    },
+    ".png",
+    True,
+    True,
+]
+
+TEST_CASE_5 = [
+    {
+        "img": torch.randint(0, 255, (1, 2, 3, 4)),
+        "img_meta_dict": {"filename_or_obj": "testfile0.nii.gz"},
+    },
+    ".nii.gz",
+    False,
+    False,
+]
+
+
+class TestSaveImaged(unittest.TestCase):
+    @parameterized.expand([TEST_CASE_0, TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5])
+    def test_saved_content(self, test_data, output_ext, resample, save_batch):
+        with tempfile.TemporaryDirectory() as tempdir:
+            trans = SaveImaged(
+                keys="img",
+                output_dir=tempdir,
+                output_ext=output_ext,
+                resample=resample,
+                save_batch=save_batch,
+            )
+            trans(test_data)
+
+            if save_batch:
+                for i in range(8):
+                    filepath = os.path.join("testfile" + str(i), "testfile" + str(i) + "_trans" + output_ext)
+                    self.assertTrue(os.path.exists(os.path.join(tempdir, filepath)))
+            else:
+                filepath = os.path.join("testfile0", "testfile0" + "_trans" + output_ext)
+                self.assertTrue(os.path.exists(os.path.join(tempdir, filepath)))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_spacing.py b/tests/test_spacing.py
index bc491f2f82..9a1ee88679 100644
--- a/tests/test_spacing.py
+++ b/tests/test_spacing.py
@@ -19,19 +19,19 @@
 
 TEST_CASES = [
     [
-        {"pixdim": (1.0, 1.5, 1.0), "padding_mode": "zeros", "dtype": np.float},
+        {"pixdim": (1.0, 1.5, 1.0), "padding_mode": "zeros", "dtype": float},
         np.arange(4).reshape((1, 2, 2)) + 1.0,  # data
         {"affine": np.eye(4)},
         np.array([[[1.0, 1.0], [3.0, 2.0]]]),
     ],
     [
-        {"pixdim": 1.0, "padding_mode": "zeros", "dtype": np.float},
+        {"pixdim": 1.0, "padding_mode": "zeros", "dtype": float},
         np.ones((1, 2, 1, 2)),  # data
         {"affine": np.eye(4)},
         np.array([[[[1.0, 1.0]], [[1.0, 1.0]]]]),
     ],
     [
-        {"pixdim": (1.0, 1.0, 1.0), "padding_mode": "zeros", "dtype": np.float},
+        {"pixdim": (1.0, 1.0, 1.0), "padding_mode": "zeros", "dtype": float},
         np.ones((1, 2, 1, 2)),  # data
         {"affine": np.eye(4)},
         np.array([[[[1.0, 1.0]], [[1.0, 1.0]]]]),
diff --git a/tests/utils.py b/tests/utils.py
index ebc9bff99f..4597a18fbd 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -47,12 +47,16 @@ def test_pretrained_networks(network, input_param, device):
     return net
 
 
+def test_is_quick():
+    return os.environ.get(quick_test_var, "").lower() == "true"
+
+
 def skip_if_quick(obj):
     """
     Skip the unit tests if environment variable `quick_test_var=true`.
     For example, the user can skip the relevant tests by setting ``export QUICKTEST=true``.
     """
-    is_quick = os.environ.get(quick_test_var, "").lower() == "true"
+    is_quick = test_is_quick()
 
     return unittest.skipIf(is_quick, "Skipping slow tests")(obj)
 
@@ -81,11 +85,11 @@ def __call__(self, obj):
         return unittest.skipIf(self.module_avail, f"Skipping because optional module present: {self.module_name}")(obj)
 
 
-def skip_if_no_cpp_extention(obj):
+def skip_if_no_cpp_extension(obj):
     """
-    Skip the unit tests if the cpp extention isnt available
+    Skip the unit tests if the cpp extension is not available
     """
-    return unittest.skipUnless(USE_COMPILED, "Skipping cpp extention tests")(obj)
+    return unittest.skipUnless(USE_COMPILED, "Skipping cpp extension tests")(obj)
 
 
 def skip_if_no_cuda(obj):
@@ -151,6 +155,19 @@ def make_nifti_image(array, affine=None):
     return image_name
 
 
+def make_rand_affine(ndim: int = 3, random_state: Optional[np.random.RandomState] = None):
+    """Create random affine transformation (with values == -1, 0 or 1)."""
+    rs = np.random if random_state is None else random_state
+
+    vals = rs.choice([-1, 1], size=ndim)
+    positions = rs.choice(range(ndim), size=ndim, replace=False)
+    af = np.zeros([ndim + 1, ndim + 1])
+    af[ndim, ndim] = 1
+    for i, (v, p) in enumerate(zip(vals, positions)):
+        af[i, p] = v
+    return af
+
+
 class DistTestCase(unittest.TestCase):
     """
     testcase without _outcome, so that it's picklable.