49 adversarial patches

examples/notebooks/image_classification_food101.ipynb

@@ -103,7 +103,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "attack = food101.create_attack(art_estimator)"
+    "attack = food101.create_pgd_attack(art_estimator)"
    ]
   },
   {

examples/src/armory/examples/image_classification/food101.py

@@ -43,6 +43,17 @@ def parse_cli_args():
         choices=["huggingface", "torchvision"],
         default="huggingface",
     )
+    parser.add_argument(
+        "--chains",
+        choices=["benign", "pgd", "patch"],
+        default=["benign", "pgd"],
+        nargs="*",
+    )
+    parser.add_argument(
+        "--patch-batch-size",
+        default=None,
+        type=int,
+    )
     return parser.parse_args()
 
 
@@ -64,8 +75,8 @@ def load_model():
         armory_model,
         loss=torch.nn.CrossEntropyLoss(),
         optimizer=torch.optim.Adam(armory_model.parameters(), lr=0.003),
-        input_shape=(224, 224, 3),
-        channels_first=False,
+        input_shape=(3, 224, 224),
+        channels_first=True,
         nb_classes=101,
         clip_values=(0.0, 1.0),
     )
@@ -170,7 +181,7 @@ def load_torchvision_dataset(
     return evaluation_dataset, labels
 
 
-def create_attack(classifier: art.estimators.classification.PyTorchClassifier):
+def create_pgd_attack(classifier: art.estimators.classification.PyTorchClassifier):
     """Creates the PGD attack"""
     pgd = armory.track.track_init_params(art.attacks.evasion.ProjectedGradientDescent)(
         classifier,
@@ -193,6 +204,34 @@ def create_attack(classifier: art.estimators.classification.PyTorchClassifier):
     return evaluation_attack
 
 
+def create_adversarial_patch_attack(
+    classifier: art.estimators.classification.PyTorchClassifier,
+    batch_size: int,
+):
+    """Creates the adversarial patch attack"""
+
+    patch = armory.track.track_init_params(art.attacks.evasion.AdversarialPatch)(
+        classifier,
+        rotation_max=22.5,
+        scale_min=0.4,
+        scale_max=1.0,
+        learning_rate=0.01,
+        max_iter=500,
+        batch_size=batch_size,
+        patch_shape=(3, 224, 224),
+    )
+
+    evaluation_attack = armory.perturbation.ArtPatchAttack(
+        name="AdversarialPatch",
+        attack=patch,
+        use_label_for_untargeted=False,
+        generate_every_batch=False,
+        apply_patch_kwargs={"scale": 0.5},
+    )
+
+    return evaluation_attack
+
+
 def create_metrics():
     """Create evaluation metrics"""
     return {
@@ -206,7 +245,16 @@ def create_metrics():
 
 
 @armory.track.track_params(prefix="main")
-def main(batch_size, export_every_n_batches, num_batches, dataset_src, seed, shuffle):
+def main(
+    batch_size,
+    export_every_n_batches,
+    num_batches,
+    dataset_src,
+    seed,
+    shuffle,
+    chains,
+    patch_batch_size,
+):
     """Perform evaluation"""
     if seed is not None:
         torch.manual_seed(seed)
@@ -218,22 +266,38 @@ def main(batch_size, export_every_n_batches, num_batches, dataset_src, seed, shu
         if dataset_src == "huggingface"
         else load_torchvision_dataset(batch_size, shuffle, sysconfig)
     )
-    attack = create_attack(art_classifier)
+    perturbations = dict()
     metrics = create_metrics()
+    profiler = armory.metrics.compute.BasicProfiler()
+
+    if "benign" in chains:
+        perturbations["benign"] = []
+
+    if "pgd" in chains:
+        pgd = create_pgd_attack(art_classifier)
+        perturbations["pgd"] = [pgd]
+
+    if "patch" in chains:
+        if patch_batch_size is None:
+            patch_batch_size = batch_size
+        patch = create_adversarial_patch_attack(
+            art_classifier, batch_size=patch_batch_size
+        )
+        perturbations["patch"] = [patch]
+
+        with profiler.measure("patch/generate"):
+            patch.generate(next(iter(dataset.dataloader)))
 
     evaluation = armory.evaluation.Evaluation(
         name=f"food101-classification-{dataset_src}",
         description=f"Image classification of food-101 from {dataset_src}",
         author="TwoSix",
         dataset=dataset,
         model=model,
-        perturbations={
-            "benign": [],
-            "attack": [attack],
-        },
+        perturbations=perturbations,
         metrics=metrics,
         exporter=armory.export.image_classification.ImageClassificationExporter(),
-        profiler=armory.metrics.compute.BasicProfiler(),
+        profiler=profiler,
         sysconfig=sysconfig,
     )
 

examples/src/armory/examples/object_detection/coco_fasterrcnn_resnet50.py

@@ -19,7 +19,6 @@
 import armory.dataset
 import armory.engine
 import armory.evaluation
-import armory.experimental.patch
 import armory.export.object_detection
 import armory.metric
 import armory.metrics.compute
@@ -129,23 +128,24 @@ def transform(sample):
     return evaluation_dataset
 
 
-def create_attack(detector):
+def create_attack(detector, batch_size: int = 1):
     dpatch = armory.track.track_init_params(art.attacks.evasion.RobustDPatch)(
         detector,
         patch_shape=(3, 50, 50),
         patch_location=(231, 231),  # middle of 512x512
-        batch_size=1,
+        batch_size=batch_size,
         sample_size=10,
         learning_rate=0.01,
         max_iter=20,
         targeted=False,
         verbose=False,
     )
 
-    evaluation_attack = armory.perturbation.ArtEvasionAttack(
+    evaluation_attack = armory.perturbation.ArtPatchAttack(
         name="RobustDPatch",
-        attack=armory.experimental.patch.AttackWrapper(dpatch),
+        attack=dpatch,
         use_label_for_untargeted=False,
+        generate_every_batch=True,
     )
 
     return evaluation_attack
@@ -172,7 +172,7 @@ def main(batch_size, export_every_n_batches, num_batches, seed, shuffle):
     model, art_detector = load_model()
 
     dataset = load_dataset(batch_size, shuffle)
-    attack = create_attack(art_detector)
+    attack = create_attack(art_detector, batch_size)
     metrics = create_metrics()
 
     evaluation = armory.evaluation.Evaluation(

examples/src/armory/examples/object_detection/license_plates_yolos.py

@@ -20,7 +20,6 @@
 import armory.dataset
 import armory.engine
 import armory.evaluation
-import armory.experimental.patch
 import armory.export.object_detection
 import armory.metric
 import armory.metrics.compute
@@ -142,23 +141,24 @@ def transform(sample):
     return evaluation_dataset
 
 
-def create_attack(detector):
+def create_attack(detector, batch_size: int = 1):
     dpatch = armory.track.track_init_params(art.attacks.evasion.RobustDPatch)(
         detector,
         patch_shape=(3, 50, 50),
         patch_location=(231, 231),  # middle of 512x512
-        batch_size=1,
+        batch_size=batch_size,
         sample_size=10,
         learning_rate=0.01,
         max_iter=20,
         targeted=False,
         verbose=False,
     )
 
-    evaluation_attack = armory.perturbation.ArtEvasionAttack(
+    evaluation_attack = armory.perturbation.ArtPatchAttack(
         name="RobustDPatch",
-        attack=armory.experimental.patch.AttackWrapper(dpatch),
+        attack=dpatch,
         use_label_for_untargeted=False,
+        generate_every_batch=True,
     )
 
     return evaluation_attack
@@ -202,7 +202,7 @@ def main(batch_size, export_every_n_batches, num_batches, seed, shuffle):
     model, art_detector = load_model()
 
     dataset = load_dataset(batch_size, shuffle)
-    attack = create_attack(art_detector)
+    attack = create_attack(art_detector, batch_size)
     metrics = create_metrics()
 
     evaluation = armory.evaluation.Evaluation(

examples/src/armory/examples/object_detection/license_plates_yolov5.py

@@ -20,7 +20,6 @@
 import armory.dataset
 import armory.engine
 import armory.evaluation
-import armory.experimental.patch
 import armory.export.object_detection
 import armory.metric
 import armory.metrics.compute
@@ -130,23 +129,24 @@ def transform(sample):
     return evaluation_dataset
 
 
-def create_attack(detector):
+def create_attack(detector, batch_size: int = 1):
     dpatch = armory.track.track_init_params(art.attacks.evasion.RobustDPatch)(
         detector,
         patch_shape=(3, 50, 50),
         patch_location=(231, 231),  # middle of 512x512
-        batch_size=1,
+        batch_size=batch_size,
         sample_size=10,
         learning_rate=0.01,
         max_iter=20,
         targeted=False,
         verbose=False,
     )
 
-    evaluation_attack = armory.perturbation.ArtEvasionAttack(
+    evaluation_attack = armory.perturbation.ArtPatchAttack(
         name="RobustDPatch",
-        attack=armory.experimental.patch.AttackWrapper(dpatch),
+        attack=dpatch,
         use_label_for_untargeted=False,
+        generate_every_batch=True,
     )
 
     return evaluation_attack
@@ -187,7 +187,7 @@ def main(batch_size, export_every_n_batches, num_batches, seed, shuffle):
     model, art_detector = load_model()
 
     dataset = load_dataset(batch_size, shuffle)
-    attack = create_attack(art_detector)
+    attack = create_attack(art_detector, batch_size)
     metrics = create_metrics()
 
     evaluation = armory.evaluation.Evaluation(

examples/src/armory/examples/utils/display.py

@@ -25,9 +25,9 @@ def get_mlflow_client():
 def get_predicted_label(filepath: Path, labels: List[str]):
     with open(filepath, "r") as infile:
         data = json.load(infile)
-        if "y_predicted" not in data:
+        if "predictions" not in data:
             return "unknown"
-        y_predicted = data["y_predicted"]
+        y_predicted = data["predictions"]
         index = np.argmax(y_predicted)
         return f"{labels[index]} ({index})"
 

library/src/armory/perturbation.py

@@ -97,27 +97,79 @@ def targeted(self) -> bool:
         """
         return self.attack.targeted
 
-    def apply(self, batch: "Batch"):
+    def _generate_y_target(self, batch: Batch) -> Optional["np.ndarray"]:
         # If targeted, use the label targeter to generate the target label
         if self.targeted:
             if TYPE_CHECKING:
                 assert self.label_targeter
-            y_target = self.label_targeter.generate(
-                self.targets_accessor.get(batch.targets)
-            )
-        else:
-            # If untargeted, use either the natural or benign labels
-            # (when set to None, the ART attack handles the benign label)
-            y_target = (
+            return self.label_targeter.generate(
                 self.targets_accessor.get(batch.targets)
-                if self.use_label_for_untargeted
-                else None
             )
 
+        # If untargeted, use either the natural or benign labels
+        # (when set to None, the ART attack handles the benign label)
+        return (
+            self.targets_accessor.get(batch.targets)
+            if self.use_label_for_untargeted
+            else None
+        )
+
+    def apply(self, batch: Batch):
+        y_target = self._generate_y_target(batch)
         perturbed = self.attack.generate(
             x=self.inputs_accessor.get(batch.inputs),
             y=y_target,
             **self.generate_kwargs,
         )
         self.inputs_accessor.set(batch.inputs, perturbed)
         batch.metadata["perturbations"][self.name] = dict(y_target=y_target)
+
+
+@dataclass
+class ArtPatchAttack(ArtEvasionAttack):
+    """
+    A perturbation using a patch evasion attack from the Adversarial Robustness
+    Toolbox (ART).
+
+    Example::
+
+        from art.attacks.evasion import AdversarialPatch
+        from charmory.perturbation import ArtPatchAttack
+
+        perturb = ArtPatchAttack(
+            name="Patch",
+            perturbation=AdversarialPatch(classifier),
+            use_label_for_untargeted=False,
+        )
+    """
+
+    generate_every_batch: bool = True
+    """Optional, whether to generate the patch for each batch """
+    apply_patch_kwargs: Dict[str, Any] = field(default_factory=dict)
+    """
+    Optional, additional keyword arguments to be used with the patch attack's
+    `apply_patch` method
+    """
+
+    def _generate(self, x: "np.ndarray", batch: Batch):
+        y_target = self._generate_y_target(batch)
+        self.patch = self.attack.generate(
+            x=x,
+            y=y_target,
+            **self.generate_kwargs,
+        )
+        batch.metadata["perturbations"][self.name] = dict(y_target=y_target)
+
+    def generate(self, batch: Batch):
+        self._generate(
+            self.inputs_accessor.get(batch.inputs),
+            batch,
+        )
+
+    def apply(self, batch: Batch):
+        x = self.inputs_accessor.get(batch.inputs)
+        if self.generate_every_batch:
+            self._generate(x, batch)
+        perturbed = self.attack.apply_patch(x=x, **self.apply_patch_kwargs)
+        self.inputs_accessor.set(batch.inputs, perturbed)
+        batch.metadata["perturbations"][self.name] = dict(patch=self.patch)