From 32b1e8f2c93bf8ca3fe3816e2cc7becbccd87609 Mon Sep 17 00:00:00 2001
From: Weilin Xu <weilin.xu@intel.com>
Date: Wed, 29 Mar 2023 11:37:42 -0700
Subject: [PATCH 01/19] Add modality-aware enforcer.

---
 mart/attack/enforcer.py                    | 23 ++++++++++++++++++++++
 mart/configs/attack/enforcer/modality.yaml |  1 +
 2 files changed, 24 insertions(+)
 create mode 100644 mart/configs/attack/enforcer/modality.yaml

diff --git a/mart/attack/enforcer.py b/mart/attack/enforcer.py
index 4d4a1364..d19d4e7c 100644
--- a/mart/attack/enforcer.py
+++ b/mart/attack/enforcer.py
@@ -11,6 +11,8 @@
 
 import torch
 
+__all__ = ["ModalityEnforcer"]
+
 
 class ConstraintViolated(Exception):
     pass
@@ -111,3 +113,24 @@ def __call__(
     ) -> None:
         for constraint in self.constraints.values():
             constraint(input_adv, input=input, target=target)
+
+
+class ModalityEnforcer(Enforcer):
+    def __init__(self, **modality_constraints: dict[str, dict[str, Constraint]]) -> None:
+        self.modality_constraints = modality_constraints
+
+    def _enforce(self, input_adv, *, input, target, modality="default"):
+        if isinstance(input_adv, torch.Tensor):
+            for constraint in self.modality_constraints[modality].values():
+                constraint(input_adv, input=input, target=target)
+        elif isinstance(input_adv, dict):
+            for modality in input_adv:
+                self._enforce(
+                    input_adv[modality], input=input[modality], target=target, modality=modality
+                )
+        elif isinstance(input_adv, list) or isinstance(input_adv, tuple):
+            for input_adv_i, input_i, target_i in zip(input_adv, input, target):
+                self._enforce(input_adv_i, input=input_i, target=target_i)
+
+    def __call__(self, input_adv, *, input, target, **kwargs):
+        self._enforce(input_adv, input=input, target=target)
diff --git a/mart/configs/attack/enforcer/modality.yaml b/mart/configs/attack/enforcer/modality.yaml
new file mode 100644
index 00000000..e5d7c6ac
--- /dev/null
+++ b/mart/configs/attack/enforcer/modality.yaml
@@ -0,0 +1 @@
+_target_: mart.attack.ModalityEnforcer

From 65f400a93fedd48c264d2b3f629e68b2e7b2ccff Mon Sep 17 00:00:00 2001
From: Weilin Xu <weilin.xu@intel.com>
Date: Wed, 29 Mar 2023 11:38:01 -0700
Subject: [PATCH 02/19] Add example.

---
 .../object_detection_rgb_mask_adversary.yaml  | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 mart/configs/attack/object_detection_rgb_mask_adversary.yaml

diff --git a/mart/configs/attack/object_detection_rgb_mask_adversary.yaml b/mart/configs/attack/object_detection_rgb_mask_adversary.yaml
new file mode 100644
index 00000000..0a627355
--- /dev/null
+++ b/mart/configs/attack/object_detection_rgb_mask_adversary.yaml
@@ -0,0 +1,22 @@
+defaults:
+  - iterative_sgd
+  - perturber: batch
+  - perturber/initializer: constant
+  - perturber/gradient_modifier: sign
+  - perturber/projector: mask_range
+  - callbacks: [progress_bar, image_visualizer]
+  - objective: zero_ap
+  - gain: rcnn_training_loss
+  - composer: overlay
+  - enforcer: modality
+  - enforcer/constraints@enforcer.rgb: [mask, pixel_range]
+
+# Make a 5-step attack for the demonstration purpose.
+optimizer:
+  lr: 55
+
+max_iters: 5
+
+perturber:
+  initializer:
+    constant: 127

From 0b46e5bfb8f82a2413e324370f610dc1e2348200 Mon Sep 17 00:00:00 2001
From: Weilin Xu <weilin.xu@intel.com>
Date: Wed, 29 Mar 2023 11:42:19 -0700
Subject: [PATCH 03/19] Backward compatible.

---
 mart/attack/enforcer.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/mart/attack/enforcer.py b/mart/attack/enforcer.py
index d19d4e7c..a1222471 100644
--- a/mart/attack/enforcer.py
+++ b/mart/attack/enforcer.py
@@ -26,6 +26,7 @@ def __call__(
         input: torch.Tensor | tuple,
         target: torch.Tensor | dict[str, Any] | tuple,
     ) -> None:
+        # TODO: Now we can get rid of this.
         if isinstance(input_adv, tuple):
             for input_adv_i, input_i, target_i in zip(input_adv, input, target):
                 self.verify(input_adv_i, input=input_i, target=target_i)
@@ -117,6 +118,10 @@ def __call__(
 
 class ModalityEnforcer(Enforcer):
     def __init__(self, **modality_constraints: dict[str, dict[str, Constraint]]) -> None:
+        # Backward compatible for existing configs without modalities.
+        if len(modality_constraints) == 1 and "constraints" in modality_constraints:
+            modality_constraints = {"default": modality_constraints}
+
         self.modality_constraints = modality_constraints
 
     def _enforce(self, input_adv, *, input, target, modality="default"):

From ff9ae095e3ff2ca8df6dd26c5cdf2e9b4d38bfa0 Mon Sep 17 00:00:00 2001
From: Weilin Xu <weilin.xu@intel.com>
Date: Wed, 29 Mar 2023 11:43:07 -0700
Subject: [PATCH 04/19] Decorator.

---
 mart/attack/enforcer.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mart/attack/enforcer.py b/mart/attack/enforcer.py
index a1222471..5a90d131 100644
--- a/mart/attack/enforcer.py
+++ b/mart/attack/enforcer.py
@@ -137,5 +137,6 @@ def _enforce(self, input_adv, *, input, target, modality="default"):
             for input_adv_i, input_i, target_i in zip(input_adv, input, target):
                 self._enforce(input_adv_i, input=input_i, target=target_i)
 
+    @torch.no_grad()
     def __call__(self, input_adv, *, input, target, **kwargs):
         self._enforce(input_adv, input=input, target=target)

From c33b116b70049027ff34dd04524cdad38f2f3486 Mon Sep 17 00:00:00 2001
From: Weilin Xu <weilin.xu@intel.com>
Date: Wed, 29 Mar 2023 11:46:00 -0700
Subject: [PATCH 05/19] Make it the default Enforcer.

---
 mart/attack/enforcer.py                       | 19 +------------------
 mart/configs/attack/enforcer/default.yaml     |  2 +-
 mart/configs/attack/enforcer/modality.yaml    |  1 -
 .../object_detection_rgb_mask_adversary.yaml  |  1 -
 4 files changed, 2 insertions(+), 21 deletions(-)
 delete mode 100644 mart/configs/attack/enforcer/modality.yaml

diff --git a/mart/attack/enforcer.py b/mart/attack/enforcer.py
index 5a90d131..f042aefa 100644
--- a/mart/attack/enforcer.py
+++ b/mart/attack/enforcer.py
@@ -11,7 +11,7 @@
 
 import torch
 
-__all__ = ["ModalityEnforcer"]
+__all__ = ["Enforcer"]
 
 
 class ConstraintViolated(Exception):
@@ -100,23 +100,6 @@ def verify(self, input_adv, *, input, target):
 
 
 class Enforcer:
-    def __init__(self, constraints: dict[str, Constraint] | None = None) -> None:
-        self.constraints = constraints or {}
-
-    @torch.no_grad()
-    def __call__(
-        self,
-        input_adv: torch.Tensor | tuple,
-        *,
-        input: torch.Tensor | tuple,
-        target: torch.Tensor | dict[str, Any] | tuple,
-        **kwargs,
-    ) -> None:
-        for constraint in self.constraints.values():
-            constraint(input_adv, input=input, target=target)
-
-
-class ModalityEnforcer(Enforcer):
     def __init__(self, **modality_constraints: dict[str, dict[str, Constraint]]) -> None:
         # Backward compatible for existing configs without modalities.
         if len(modality_constraints) == 1 and "constraints" in modality_constraints:
diff --git a/mart/configs/attack/enforcer/default.yaml b/mart/configs/attack/enforcer/default.yaml
index 885e0860..94ed9441 100644
--- a/mart/configs/attack/enforcer/default.yaml
+++ b/mart/configs/attack/enforcer/default.yaml
@@ -1,4 +1,4 @@
 defaults:
   - constraints: null
 
-_target_: mart.attack.enforcer.Enforcer
+_target_: mart.attack.Enforcer
diff --git a/mart/configs/attack/enforcer/modality.yaml b/mart/configs/attack/enforcer/modality.yaml
deleted file mode 100644
index e5d7c6ac..00000000
--- a/mart/configs/attack/enforcer/modality.yaml
+++ /dev/null
@@ -1 +0,0 @@
-_target_: mart.attack.ModalityEnforcer
diff --git a/mart/configs/attack/object_detection_rgb_mask_adversary.yaml b/mart/configs/attack/object_detection_rgb_mask_adversary.yaml
index 0a627355..cbd0ee80 100644
--- a/mart/configs/attack/object_detection_rgb_mask_adversary.yaml
+++ b/mart/configs/attack/object_detection_rgb_mask_adversary.yaml
@@ -8,7 +8,6 @@ defaults:
   - objective: zero_ap
   - gain: rcnn_training_loss
   - composer: overlay
-  - enforcer: modality
   - enforcer/constraints@enforcer.rgb: [mask, pixel_range]
 
 # Make a 5-step attack for the demonstration purpose.

From 8630946dcf2453a738e2f15686aeca2afe9808c2 Mon Sep 17 00:00:00 2001
From: Weilin Xu <weilin.xu@intel.com>
Date: Wed, 29 Mar 2023 11:52:01 -0700
Subject: [PATCH 06/19] Fix backward compatibility.

---
 mart/attack/enforcer.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/mart/attack/enforcer.py b/mart/attack/enforcer.py
index f042aefa..2cae8bc4 100644
--- a/mart/attack/enforcer.py
+++ b/mart/attack/enforcer.py
@@ -101,13 +101,10 @@ def verify(self, input_adv, *, input, target):
 
 class Enforcer:
     def __init__(self, **modality_constraints: dict[str, dict[str, Constraint]]) -> None:
-        # Backward compatible for existing configs without modalities.
-        if len(modality_constraints) == 1 and "constraints" in modality_constraints:
-            modality_constraints = {"default": modality_constraints}
-
         self.modality_constraints = modality_constraints
 
-    def _enforce(self, input_adv, *, input, target, modality="default"):
+    def _enforce(self, input_adv, *, input, target, modality="constraints"):
+        # Set modality="constraitns" by default, so that it is backward compatible with existing configs without modalities.
         if isinstance(input_adv, torch.Tensor):
             for constraint in self.modality_constraints[modality].values():
                 constraint(input_adv, input=input, target=target)

From cec47b65d9e22b0c1d7ca373d8b0775b03ee0b25 Mon Sep 17 00:00:00 2001
From: Weilin Xu <weilin.xu@intel.com>
Date: Wed, 29 Mar 2023 11:54:48 -0700
Subject: [PATCH 07/19] Remove previous tuple-aware mechanism.

---
 mart/attack/enforcer.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/mart/attack/enforcer.py b/mart/attack/enforcer.py
index 2cae8bc4..ce67ac68 100644
--- a/mart/attack/enforcer.py
+++ b/mart/attack/enforcer.py
@@ -26,12 +26,7 @@ def __call__(
         input: torch.Tensor | tuple,
         target: torch.Tensor | dict[str, Any] | tuple,
     ) -> None:
-        # TODO: Now we can get rid of this.
-        if isinstance(input_adv, tuple):
-            for input_adv_i, input_i, target_i in zip(input_adv, input, target):
-                self.verify(input_adv_i, input=input_i, target=target_i)
-        else:
-            self.verify(input_adv, input=input, target=target)
+        self.verify(input_adv, input=input, target=target)
 
     @abc.abstractmethod
     def verify(

From 6955448861061f359b25a961a37ec1b8cd1f2f74 Mon Sep 17 00:00:00 2001
From: Weilin Xu <weilin.xu@intel.com>
Date: Wed, 29 Mar 2023 11:56:40 -0700
Subject: [PATCH 08/19] Fix example.

---
 mart/configs/attack/object_detection_rgb_mask_adversary.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mart/configs/attack/object_detection_rgb_mask_adversary.yaml b/mart/configs/attack/object_detection_rgb_mask_adversary.yaml
index cbd0ee80..a2bb039e 100644
--- a/mart/configs/attack/object_detection_rgb_mask_adversary.yaml
+++ b/mart/configs/attack/object_detection_rgb_mask_adversary.yaml
@@ -8,6 +8,7 @@ defaults:
   - objective: zero_ap
   - gain: rcnn_training_loss
   - composer: overlay
+  - enforcer: default
   - enforcer/constraints@enforcer.rgb: [mask, pixel_range]
 
 # Make a 5-step attack for the demonstration purpose.

From 0e9c61e322afed76479c7eb0ddb8c0e603f5a2b3 Mon Sep 17 00:00:00 2001
From: Weilin Xu <weilin.xu@intel.com>
Date: Wed, 29 Mar 2023 12:01:44 -0700
Subject: [PATCH 09/19] Fix typo.

---
 mart/attack/enforcer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mart/attack/enforcer.py b/mart/attack/enforcer.py
index ce67ac68..a46a89c2 100644
--- a/mart/attack/enforcer.py
+++ b/mart/attack/enforcer.py
@@ -99,7 +99,7 @@ def __init__(self, **modality_constraints: dict[str, dict[str, Constraint]]) ->
         self.modality_constraints = modality_constraints
 
     def _enforce(self, input_adv, *, input, target, modality="constraints"):
-        # Set modality="constraitns" by default, so that it is backward compatible with existing configs without modalities.
+        # Set modality="constraints" by default, so that it is backward compatible with existing configs without modalities.
         if isinstance(input_adv, torch.Tensor):
             for constraint in self.modality_constraints[modality].values():
                 constraint(input_adv, input=input, target=target)

From f30e19e6e1d39393a8ef6ac97aaf01c89acd6e66 Mon Sep 17 00:00:00 2001
From: Weilin Xu <weilin.xu@intel.com>
Date: Wed, 29 Mar 2023 12:05:38 -0700
Subject: [PATCH 10/19] Comment.

---
 mart/attack/enforcer.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/mart/attack/enforcer.py b/mart/attack/enforcer.py
index a46a89c2..19cfa3bc 100644
--- a/mart/attack/enforcer.py
+++ b/mart/attack/enforcer.py
@@ -101,14 +101,17 @@ def __init__(self, **modality_constraints: dict[str, dict[str, Constraint]]) ->
     def _enforce(self, input_adv, *, input, target, modality="constraints"):
         # Set modality="constraints" by default, so that it is backward compatible with existing configs without modalities.
         if isinstance(input_adv, torch.Tensor):
+            # Finally we can verify constraints on tensor, per its modality.
             for constraint in self.modality_constraints[modality].values():
                 constraint(input_adv, input=input, target=target)
         elif isinstance(input_adv, dict):
+            # The dict input has modalities specified in keys, passing them recursively.
             for modality in input_adv:
                 self._enforce(
                     input_adv[modality], input=input[modality], target=target, modality=modality
                 )
         elif isinstance(input_adv, list) or isinstance(input_adv, tuple):
+            # The list or tuple input is a collection of sub-input.
             for input_adv_i, input_i, target_i in zip(input_adv, input, target):
                 self._enforce(input_adv_i, input=input_i, target=target_i)
 

From 66ad8c4930eecd988074df282e8a7c62b575488a Mon Sep 17 00:00:00 2001
From: Weilin Xu <weilin.xu@intel.com>
Date: Wed, 29 Mar 2023 15:42:10 -0700
Subject: [PATCH 11/19] Fix type annotation.

---
 mart/attack/enforcer.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/mart/attack/enforcer.py b/mart/attack/enforcer.py
index 19cfa3bc..f4fd3be3 100644
--- a/mart/attack/enforcer.py
+++ b/mart/attack/enforcer.py
@@ -21,10 +21,10 @@ class ConstraintViolated(Exception):
 class Constraint(abc.ABC):
     def __call__(
         self,
-        input_adv: torch.Tensor | tuple,
+        input_adv: torch.Tensor,
         *,
-        input: torch.Tensor | tuple,
-        target: torch.Tensor | dict[str, Any] | tuple,
+        input: torch.Tensor,
+        target: torch.Tensor | dict[str, Any],
     ) -> None:
         self.verify(input_adv, input=input, target=target)
 

From d566a54a4af1cbacb7ab982513a6df4f3dd203f2 Mon Sep 17 00:00:00 2001
From: Weilin Xu <weilin.xu@intel.com>
Date: Wed, 29 Mar 2023 16:01:57 -0700
Subject: [PATCH 12/19] Move recursion to __call__().

---
 mart/attack/enforcer.py | 37 +++++++++++++++++++++++++------------
 1 file changed, 25 insertions(+), 12 deletions(-)

diff --git a/mart/attack/enforcer.py b/mart/attack/enforcer.py
index f4fd3be3..6692a89d 100644
--- a/mart/attack/enforcer.py
+++ b/mart/attack/enforcer.py
@@ -98,23 +98,36 @@ class Enforcer:
     def __init__(self, **modality_constraints: dict[str, dict[str, Constraint]]) -> None:
         self.modality_constraints = modality_constraints
 
-    def _enforce(self, input_adv, *, input, target, modality="constraints"):
-        # Set modality="constraints" by default, so that it is backward compatible with existing configs without modalities.
+    @torch.no_grad()
+    def _enforce(
+        self,
+        input_adv: torch.Tensor,
+        *,
+        input: torch.Tensor,
+        target: torch.Tensor | dict[str, Any],
+        modality: str,
+    ):
+        for constraint in self.modality_constraints[modality].values():
+            constraint(input_adv, input=input, target=target)
+
+    def __call__(
+        self,
+        input_adv: torch.Tensor | tuple[torch.Tensor] | dict[str, torch.Tensor],
+        *,
+        input: torch.Tensor | tuple[torch.Tensor] | dict[str, torch.Tensor],
+        target: torch.Tensor | dict[str, Any],
+        modality: str = "constraints",
+        **kwargs,
+    ):
         if isinstance(input_adv, torch.Tensor):
             # Finally we can verify constraints on tensor, per its modality.
-            for constraint in self.modality_constraints[modality].values():
-                constraint(input_adv, input=input, target=target)
+            # Set modality="constraints" by default, so that it is backward compatible with existing configs without modalities.
+            self._enforce(input_adv, input=input, target=target, modality=modality)
         elif isinstance(input_adv, dict):
             # The dict input has modalities specified in keys, passing them recursively.
             for modality in input_adv:
-                self._enforce(
-                    input_adv[modality], input=input[modality], target=target, modality=modality
-                )
+                self(input_adv[modality], input=input[modality], target=target, modality=modality)
         elif isinstance(input_adv, list) or isinstance(input_adv, tuple):
             # The list or tuple input is a collection of sub-input.
             for input_adv_i, input_i, target_i in zip(input_adv, input, target):
-                self._enforce(input_adv_i, input=input_i, target=target_i)
-
-    @torch.no_grad()
-    def __call__(self, input_adv, *, input, target, **kwargs):
-        self._enforce(input_adv, input=input, target=target)
+                self(input_adv_i, input=input_i, target=target_i)

From ea261053112701560acd923c5f4b0c05590d8d08 Mon Sep 17 00:00:00 2001
From: Weilin Xu <weilin.xu@intel.com>
Date: Wed, 29 Mar 2023 16:04:07 -0700
Subject: [PATCH 13/19] Simplify condition experession.

---
 mart/attack/enforcer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mart/attack/enforcer.py b/mart/attack/enforcer.py
index 6692a89d..150ac8d0 100644
--- a/mart/attack/enforcer.py
+++ b/mart/attack/enforcer.py
@@ -127,7 +127,7 @@ def __call__(
             # The dict input has modalities specified in keys, passing them recursively.
             for modality in input_adv:
                 self(input_adv[modality], input=input[modality], target=target, modality=modality)
-        elif isinstance(input_adv, list) or isinstance(input_adv, tuple):
+        elif isinstance(input_adv, (list, tuple)):
             # The list or tuple input is a collection of sub-input.
             for input_adv_i, input_i, target_i in zip(input_adv, input, target):
                 self(input_adv_i, input=input_i, target=target_i)

From 90daa4dfa64cdad90c80a8231887823ac9568dc2 Mon Sep 17 00:00:00 2001
From: Weilin Xu <weilin.xu@intel.com>
Date: Wed, 29 Mar 2023 16:04:49 -0700
Subject: [PATCH 14/19] Type annotation.

---
 mart/attack/enforcer.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mart/attack/enforcer.py b/mart/attack/enforcer.py
index 150ac8d0..f92baa56 100644
--- a/mart/attack/enforcer.py
+++ b/mart/attack/enforcer.py
@@ -112,9 +112,9 @@ def _enforce(
 
     def __call__(
         self,
-        input_adv: torch.Tensor | tuple[torch.Tensor] | dict[str, torch.Tensor],
+        input_adv: torch.Tensor | tuple | list[torch.Tensor] | dict[str, torch.Tensor],
         *,
-        input: torch.Tensor | tuple[torch.Tensor] | dict[str, torch.Tensor],
+        input: torch.Tensor | tuple | list[torch.Tensor] | dict[str, torch.Tensor],
         target: torch.Tensor | dict[str, Any],
         modality: str = "constraints",
         **kwargs,

From abdedc830b9cf77c480269c85d793ca6447f235e Mon Sep 17 00:00:00 2001
From: Weilin Xu <weilin.xu@intel.com>
Date: Wed, 29 Mar 2023 16:16:44 -0700
Subject: [PATCH 15/19] Test Enforcer().

---
 tests/test_enforcer.py | 49 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)

diff --git a/tests/test_enforcer.py b/tests/test_enforcer.py
index 78053b53..2a77981a 100644
--- a/tests/test_enforcer.py
+++ b/tests/test_enforcer.py
@@ -7,7 +7,7 @@
 import pytest
 import torch
 
-from mart.attack.enforcer import ConstraintViolated, Integer, Lp, Mask, Range
+from mart.attack.enforcer import ConstraintViolated, Enforcer, Integer, Lp, Mask, Range
 
 
 def test_constraint_range():
@@ -67,3 +67,50 @@ def test_constraint_mask():
     constraint(input + perturbation * mask, input=input, target=target)
     with pytest.raises(ConstraintViolated):
         constraint(input + perturbation, input=input, target=target)
+
+
+def test_enforcer_non_modality():
+    enforcer = Enforcer(constraints={"range": Range(min=0, max=255)})
+
+    input = torch.tensor([0, 0, 0])
+    perturbation = torch.tensor([0, 128, 255])
+    input_adv = input + perturbation
+    target = None
+
+    # tensor input.
+    enforcer(input_adv, input=input, target=target)
+    # list of tensor input.
+    enforcer([input_adv], input=[input], target=[target])
+
+    perturbation = torch.tensor([0, -1, 255])
+    input_adv = input + perturbation
+
+    with pytest.raises(ConstraintViolated):
+        enforcer(input_adv, input=input, target=target)
+
+    with pytest.raises(ConstraintViolated):
+        enforcer([input_adv], input=[input], target=[target])
+
+
+def test_enforcer_modality():
+    # Assume a rgb modality.
+    enforcer = Enforcer(rgb={"range": Range(min=0, max=255)})
+
+    input = torch.tensor([0, 0, 0])
+    perturbation = torch.tensor([0, 128, 255])
+    input_adv = input + perturbation
+    target = None
+
+    # Dictionary input.
+    enforcer({"rgb": input_adv}, input={"rgb": input}, target=target)
+    # List of dictionary input.
+    enforcer([{"rgb": input_adv}], input=[{"rgb": input}], target=[target])
+
+    perturbation = torch.tensor([0, -1, 255])
+    input_adv = input + perturbation
+
+    with pytest.raises(ConstraintViolated):
+        enforcer({"rgb": input_adv}, input={"rgb": input}, target=target)
+
+    with pytest.raises(ConstraintViolated):
+        enforcer([{"rgb": input_adv}], input=[{"rgb": input}], target=[target])

From 7c6afe3329e9256070872a22cdeda88271a4c143 Mon Sep 17 00:00:00 2001
From: Weilin Xu <weilin.xu@intel.com>
Date: Wed, 29 Mar 2023 16:18:57 -0700
Subject: [PATCH 16/19] Comment.

---
 mart/attack/enforcer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mart/attack/enforcer.py b/mart/attack/enforcer.py
index f92baa56..a6e51955 100644
--- a/mart/attack/enforcer.py
+++ b/mart/attack/enforcer.py
@@ -128,6 +128,6 @@ def __call__(
             for modality in input_adv:
                 self(input_adv[modality], input=input[modality], target=target, modality=modality)
         elif isinstance(input_adv, (list, tuple)):
-            # The list or tuple input is a collection of sub-input.
+            # The list or tuple input is a collection of sub-input and sub-target.
             for input_adv_i, input_i, target_i in zip(input_adv, input, target):
                 self(input_adv_i, input=input_i, target=target_i)

From d86a71cc05a94903ce5f2cbb99a5c3d8906004e2 Mon Sep 17 00:00:00 2001
From: Weilin Xu <weilin.xu@intel.com>
Date: Thu, 30 Mar 2023 10:49:32 -0700
Subject: [PATCH 17/19] Add tuple tests.

---
 tests/test_enforcer.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tests/test_enforcer.py b/tests/test_enforcer.py
index 2a77981a..2c56b3ad 100644
--- a/tests/test_enforcer.py
+++ b/tests/test_enforcer.py
@@ -81,6 +81,8 @@ def test_enforcer_non_modality():
     enforcer(input_adv, input=input, target=target)
     # list of tensor input.
     enforcer([input_adv], input=[input], target=[target])
+    # tuple of tensor input.
+    enforcer((input_adv,), input=(input,), target=(target,))
 
     perturbation = torch.tensor([0, -1, 255])
     input_adv = input + perturbation
@@ -91,6 +93,9 @@ def test_enforcer_non_modality():
     with pytest.raises(ConstraintViolated):
         enforcer([input_adv], input=[input], target=[target])
 
+    with pytest.raises(ConstraintViolated):
+        enforcer((input_adv,), input=(input,), target=(target,))
+
 
 def test_enforcer_modality():
     # Assume a rgb modality.
@@ -105,6 +110,8 @@ def test_enforcer_modality():
     enforcer({"rgb": input_adv}, input={"rgb": input}, target=target)
     # List of dictionary input.
     enforcer([{"rgb": input_adv}], input=[{"rgb": input}], target=[target])
+    # Tuple of dictionary input.
+    enforcer(({"rgb": input_adv},), input=({"rgb": input},), target=(target,))
 
     perturbation = torch.tensor([0, -1, 255])
     input_adv = input + perturbation
@@ -114,3 +121,6 @@ def test_enforcer_modality():
 
     with pytest.raises(ConstraintViolated):
         enforcer([{"rgb": input_adv}], input=[{"rgb": input}], target=[target])
+
+    with pytest.raises(ConstraintViolated):
+        enforcer(({"rgb": input_adv},), input=({"rgb": input},), target=(target,))

From 66be761a5cf3e53887be5576d2755d6472234203 Mon Sep 17 00:00:00 2001
From: Weilin Xu <weilin.xu@intel.com>
Date: Thu, 30 Mar 2023 10:52:47 -0700
Subject: [PATCH 18/19] Strengthen data type checking.

---
 mart/attack/enforcer.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/mart/attack/enforcer.py b/mart/attack/enforcer.py
index a6e51955..1fee5f30 100644
--- a/mart/attack/enforcer.py
+++ b/mart/attack/enforcer.py
@@ -119,6 +119,8 @@ def __call__(
         modality: str = "constraints",
         **kwargs,
     ):
+        assert type(input_adv) == type(input)
+
         if isinstance(input_adv, torch.Tensor):
             # Finally we can verify constraints on tensor, per its modality.
             # Set modality="constraints" by default, so that it is backward compatible with existing configs without modalities.
@@ -131,3 +133,5 @@ def __call__(
             # The list or tuple input is a collection of sub-input and sub-target.
             for input_adv_i, input_i, target_i in zip(input_adv, input, target):
                 self(input_adv_i, input=input_i, target=target_i)
+        else:
+            raise ValueError(f"Unsupported data type of input_adv: {type(input_adv)}.")

From e7a26d8ca59fa3795d95c35e8e6a5640d0a6e79e Mon Sep 17 00:00:00 2001
From: Weilin Xu <weilin.xu@intel.com>
Date: Thu, 30 Mar 2023 11:40:13 -0700
Subject: [PATCH 19/19] Add logic checking.

---
 mart/attack/enforcer.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/mart/attack/enforcer.py b/mart/attack/enforcer.py
index 1fee5f30..babc44e6 100644
--- a/mart/attack/enforcer.py
+++ b/mart/attack/enforcer.py
@@ -130,8 +130,10 @@ def __call__(
             for modality in input_adv:
                 self(input_adv[modality], input=input[modality], target=target, modality=modality)
         elif isinstance(input_adv, (list, tuple)):
+            # We assume a modality-dictionary only contains tensors, but not list/tuple.
+            assert modality == "constraints"
             # The list or tuple input is a collection of sub-input and sub-target.
             for input_adv_i, input_i, target_i in zip(input_adv, input, target):
-                self(input_adv_i, input=input_i, target=target_i)
+                self(input_adv_i, input=input_i, target=target_i, modality=modality)
         else:
             raise ValueError(f"Unsupported data type of input_adv: {type(input_adv)}.")