common transforms

.github/workflows/ci.yml

@@ -94,7 +94,7 @@ jobs:
             script: mnist_vit_pgd.py --batch-size 2 --num-batches 1 --export-every-n-batches 1
           - name: Object Detection
             dir: src/charmory_examples/object_detection
-            script: yolov5_license_plates.py --batch-size 2 --num-batches 1 --export-every-n-batches 1
+            script: license_plates_yolov5_robustdpatch.py --batch-size 2 --num-batches 1 --export-every-n-batches 1
         python-version: ["3.8"]
     steps:
       - name: Checkout

examples/src/charmory_examples/image_classification/cifar10_jatic_toolbox.py

@@ -5,11 +5,11 @@
 interoperability between dataset and model providers. This file is NOT
 a good example of using the JATIC toolbox or Armory.
 """
-import argparse
 import sys
 
 import art.attacks.evasion
 from art.estimators.classification import PyTorchClassifier
+from charmory_examples.utils.args import create_parser
 from jatic_toolbox import __version__ as jatic_version
 from jatic_toolbox import load_dataset as load_jatic_dataset
 from jatic_toolbox import load_model as load_jatic_model
@@ -29,10 +29,8 @@
 from charmory.track import track_init_params, track_params
 from charmory.utils import create_jatic_dataset_transform
 
-BATCH_SIZE = 16
 
-
-def load_huggingface_dataset(transform):
+def load_huggingface_dataset(transform, batch_size):
     print("Loading HuggingFace dataset from jatic_toolbox")
 
     train_dataset = track_params(load_jatic_dataset)(
@@ -44,7 +42,7 @@ def load_huggingface_dataset(transform):
     train_dataset.set_transform(transform)
     train_dataloader = ArmoryDataLoader(
         dataset=train_dataset,
-        batch_size=BATCH_SIZE,
+        batch_size=batch_size,
         shuffle=True,
     )
 
@@ -57,14 +55,14 @@ def load_huggingface_dataset(transform):
     test_dataset.set_transform(transform)
     test_dataloader = ArmoryDataLoader(
         dataset=test_dataset,
-        batch_size=BATCH_SIZE,
+        batch_size=batch_size,
         shuffle=False,
     )
 
     return train_dataloader, test_dataloader
 
 
-def load_torchvision_dataset(transform):
+def load_torchvision_dataset(transform, batch_size):
     print("Loading torchvision dataset from jatic_toolbox")
     train_dataset = track_params(load_jatic_dataset)(
         provider="torchvision",
@@ -77,7 +75,7 @@ def load_torchvision_dataset(transform):
     train_dataset.set_transform(transform)
     train_dataloader = ArmoryDataLoader(
         dataset=train_dataset,
-        batch_size=BATCH_SIZE,
+        batch_size=batch_size,
         shuffle=True,
     )
 
@@ -92,7 +90,7 @@ def load_torchvision_dataset(transform):
     test_dataset.set_transform(transform)
     test_dataloader = ArmoryDataLoader(
         dataset=test_dataset,
-        batch_size=BATCH_SIZE,
+        batch_size=batch_size,
         shuffle=False,
     )
 
@@ -146,9 +144,11 @@ def load_torchvision_model():
 
 
 def main():
-    parser = argparse.ArgumentParser(
+    parser = create_parser(
         description="Run example using models and datasets from the JATIC toolbox",
-        formatter_class=argparse.RawTextHelpFormatter,
+        batch_size=16,
+        export_every_n_batches=5,
+        num_batches=5,
     )
     parser.add_argument(
         "--dataset",
@@ -177,9 +177,13 @@ def main():
         loaded_model, transform = load_huggingface_model()
 
     if args.dataset == "torchvision":
-        train_dataset, test_dataset = load_torchvision_dataset(transform)
+        train_dataset, test_dataset = load_torchvision_dataset(
+            transform, args.batch_size
+        )
     else:
-        train_dataset, test_dataset = load_huggingface_dataset(transform)
+        train_dataset, test_dataset = load_huggingface_dataset(
+            transform, args.batch_size
+        )
 
     dataset = Dataset(
         name="CIFAR10",
@@ -228,9 +232,11 @@ def main():
         metric=metric,
     )
 
-    task = ImageClassificationTask(evaluation, num_classes=10, export_every_n_batches=5)
+    task = ImageClassificationTask(
+        evaluation, num_classes=10, export_every_n_batches=args.export_every_n_batches
+    )
 
-    engine = EvaluationEngine(task, limit_test_batches=5)
+    engine = EvaluationEngine(task, limit_test_batches=args.num_batches)
     results = engine.run()
     print_outputs(dataset, model, results)
 

examples/src/charmory_examples/image_classification/eurosat_precomputed_pgd/evaluation.py

@@ -1,13 +1,13 @@
 """Definition for the EuroSAT classification evaluation"""
 
-import argparse
 from copy import deepcopy
 import os
 from typing import Optional
 
 import albumentations as A
 from art.attacks.evasion import ProjectedGradientDescent
 from art.estimators.classification import PyTorchClassifier
+from charmory_examples.utils.args import create_parser
 import datasets
 import jatic_toolbox
 from jatic_toolbox.interop.huggingface import HuggingFaceVisionDataset
@@ -32,9 +32,10 @@
 
 def get_cli_args(with_attack: bool):
     """Get CLI-specified arguments to configure the evaluation."""
-    parser = argparse.ArgumentParser(
+    parser = create_parser(
         description="Run EuroSAT image classification evaluation",
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        batch_size=4,
+        export_every_n_batches=5,
     )
     parser.add_argument(
         "model_name",
@@ -45,21 +46,6 @@ def get_cli_args(with_attack: bool):
             "dataset_path",
             type=str,
         )
-    parser.add_argument(
-        "--batch-size",
-        default=4,
-        type=int,
-    )
-    parser.add_argument(
-        "--num-batches",
-        default=None,
-        type=int,
-    )
-    parser.add_argument(
-        "--export-every-n-batches",
-        default=5,
-        type=int,
-    )
 
     if with_attack:
         attack_args = parser.add_argument_group("attack", "PGD attack parameters")

examples/src/charmory_examples/image_classification/food101/food101_pgd.py

@@ -6,7 +6,6 @@
 
 import armory.baseline_models.pytorch.food101
 import armory.data.datasets
-from armory.instrument.config import MetricsLogger
 from armory.metrics.compute import BasicProfiler
 import armory.version
 from charmory.data import ArmoryDataLoader
@@ -106,13 +105,6 @@ def main():
 
     metric = Metric(
         profiler=BasicProfiler(),
-        logger=MetricsLogger(
-            supported_metrics=["accuracy"],
-            perturbation=["linf"],
-            task=["categorical_accuracy"],
-            means=True,
-            record_metric_per_sample=False,
-        ),
     )
 
     evaluation = Evaluation(

examples/src/charmory_examples/image_classification/imagenet1k_resnet34_pgd.py

@@ -17,10 +17,10 @@
 from charmory.engine import EvaluationEngine
 from charmory.evaluation import Attack, Dataset, Evaluation, Metric, Model
 from charmory.experimental.example_results import print_outputs
+from charmory.experimental.transforms import create_image_classification_transform
 from charmory.model.image_classification import JaticImageClassificationModel
 from charmory.tasks.image_classification import ImageClassificationTask
 from charmory.track import track_init_params, track_params
-from charmory.utils import PILtoNumpy_HuggingFace_Variable_Length
 
 BATCH_SIZE = 16
 
@@ -29,7 +29,9 @@
 
 
 def load_huggingface_dataset():
-    transform = PILtoNumpy_HuggingFace_Variable_Length(size=(500, 500))
+    transform = create_image_classification_transform(
+        max_size=500,
+    )
     train_dataset = load_jatic_dataset(
         provider="huggingface",
         dataset_name="imagenet-1k",

examples/src/charmory_examples/image_classification/kaludi_food_category_pgd.py

@@ -1,8 +1,8 @@
-import argparse
 from pprint import pprint
 
 import art.attacks.evasion
 from art.estimators.classification import PyTorchClassifier
+from charmory_examples.utils.args import create_parser
 import jatic_toolbox
 import numpy as np
 import torch.nn
@@ -19,24 +19,11 @@
 
 
 def get_cli_args():
-    parser = argparse.ArgumentParser(
+    parser = create_parser(
         description="Run food classification example using models and datasets from the JATIC toolbox",
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-    )
-    parser.add_argument(
-        "--batch-size",
-        default=16,
-        type=int,
-    )
-    parser.add_argument(
-        "--export-every-n-batches",
-        default=5,
-        type=int,
-    )
-    parser.add_argument(
-        "--num-batches",
-        default=5,
-        type=int,
+        batch_size=16,
+        export_every_n_batches=5,
+        num_batches=5,
     )
     return parser.parse_args()
 

examples/src/charmory_examples/image_classification/pokemon_pgd.py

@@ -15,9 +15,9 @@
 from charmory.engine import EvaluationEngine
 from charmory.evaluation import Attack, Dataset, Evaluation, Metric, Model
 from charmory.experimental.example_results import print_outputs
+from charmory.experimental.transforms import create_image_classification_transform
 from charmory.tasks.image_classification import ImageClassificationTask
 from charmory.track import track_init_params, track_params
-from charmory.utils import PILtoNumpy_HuggingFace
 
 BATCH_SIZE = 16
 
@@ -26,7 +26,9 @@
 
 
 def load_huggingface_dataset():
-    transform = PILtoNumpy_HuggingFace()
+    transform = create_image_classification_transform(
+        image_from_np=lambda img: img,
+    )
 
     train_dataset = track_params(load_jatic_dataset)(
         provider="huggingface",

examples/src/charmory_examples/object_detection/coco_fasterrcnn_resnet50_robustdpatch.py

@@ -1,9 +1,9 @@
-import argparse
+from pprint import pprint
 
 from PIL import Image
-import albumentations as A
 import art.attacks.evasion
 from art.estimators.object_detection import PyTorchFasterRCNN
+from charmory_examples.utils.args import create_parser
 import jatic_toolbox
 import numpy as np
 
@@ -12,32 +12,22 @@
 from charmory.data import ArmoryDataLoader
 from charmory.engine import EvaluationEngine
 from charmory.evaluation import Attack, Dataset, Evaluation, Metric, Model
-from charmory.experimental.example_results import print_outputs
+from charmory.experimental.transforms import (
+    BboxFormat,
+    create_object_detection_transform,
+)
 from charmory.model.object_detection import JaticObjectDetectionModel
 from charmory.tasks.object_detection import ObjectDetectionTask
 from charmory.track import track_init_params, track_params
 from charmory.utils import create_jatic_dataset_transform
 
 
 def get_cli_args():
-    parser = argparse.ArgumentParser(
+    parser = create_parser(
         description="Run COCO object detection example using models and datasets from the JATIC toolbox",
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-    )
-    parser.add_argument(
-        "--batch-size",
-        default=4,
-        type=int,
-    )
-    parser.add_argument(
-        "--export-every-n-batches",
-        default=5,
-        type=int,
-    )
-    parser.add_argument(
-        "--num-batches",
-        default=20,
-        type=int,
+        batch_size=4,
+        export_every_n_batches=5,
+        num_batches=20,
     )
     return parser.parse_args()
 
@@ -85,42 +75,16 @@ def filter(sample):
 
     model_transform = create_jatic_dataset_transform(model.preprocessor)
 
-    img_transforms = A.Compose(
-        [
-            A.LongestMaxSize(max_size=400),
-            A.PadIfNeeded(
-                min_height=400,
-                min_width=400,
-                border_mode=0,
-                value=(0, 0, 0),
-            ),
-        ],
-        bbox_params=A.BboxParams(
-            format="pascal_voc",
-            label_fields=["labels"],
-        ),
+    dataset.set_transform(
+        create_object_detection_transform(
+            image_from_np=Image.fromarray,
+            max_size=400,
+            format=BboxFormat.XYXY,
+            label_fields=["category"],
+            postprocessor=model_transform,
+        )
     )
 
-    def transform(sample):
-        transformed = dict(image=[], objects=[])
-        for i in range(len(sample["image"])):
-            transformed_img = img_transforms(
-                image=np.asarray(sample["image"][i]),
-                bboxes=sample["objects"][i]["bbox"],
-                labels=sample["objects"][i]["category"],
-            )
-            transformed["image"].append(Image.fromarray(transformed_img["image"]))
-            transformed["objects"].append(
-                dict(
-                    bbox=transformed_img["bboxes"],
-                    category=transformed_img["labels"],
-                )
-            )
-        transformed = model_transform(transformed)
-        return transformed
-
-    dataset.set_transform(transform)
-
     dataloader = ArmoryDataLoader(dataset, batch_size=args.batch_size)
 
     ###
@@ -178,7 +142,7 @@ def transform(sample):
     )
     engine = EvaluationEngine(task, limit_test_batches=args.num_batches)
     results = engine.run()
-    print_outputs(dataset, model, results)
+    pprint(results)
 
     print("JATIC Experiment Complete!")
     return 0