Merge pull request #25 from Grutschus/16-setup-a-videomaev2-model

16 setup a videomaev2 model

.dvc/config

@@ -1,6 +1,7 @@
 [core]
     remote = storage
     autostage = true
+    hardlink_lock = true
 ['remote "storage"']
     url = s3://human-fall-detection/
     endpointurl = https://s3.tebi.io

.gitignore

@@ -158,3 +158,6 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+# Container files
+**.sif

.vscode/settings.json

@@ -9,5 +9,6 @@
         "tests"
     ],
     "python.testing.unittestEnabled": false,
-    "python.testing.pytestEnabled": true
+    "python.testing.pytestEnabled": true,
+    "jupyter.debugJustMyCode": false
 }

README.md

@@ -69,8 +69,8 @@ However, if you want to directly access the data we use, follow these steps.
 To authenticate with our bucket, add the credentials to your **local** DVC config:
 
 ```bash
-dvc remote modify storage access_key_id [ACCESS_KEY_ID]
-dvc remote modify storage secret_access_key [SECRET_ACCESS_KEY]
+dvc remote modify --local storage access_key_id [ACCESS_KEY_ID]
+dvc remote modify --local storage secret_access_key [SECRET_ACCESS_KEY]
 ```
 
 Alternatively, you can set the environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` in your terminal.

configs/datasets/ds_uniformsample_existencelabel.py

@@ -0,0 +1,22 @@
+from datasets.transforms.label_strategy import HQFD_LABEL_DESCRIPTION
+
+custom_imports = dict(imports="datasets", allow_failed_imports=False)
+type = "HighQualityFallDataset"
+sampling_strategy = dict(type="UniformSampling", clip_len=10)
+label_strategy = dict(type="ExistenceLabel", label_description=HQFD_LABEL_DESCRIPTION)
+ann_file = "tests/test_data/test_annotation.csv"
+pipeline = [
+    dict(type="DecordInit"),
+    dict(type="ClipVideo"),
+    dict(type="SampleFrames", clip_len=16, frame_interval=4, num_clips=1),
+    dict(type="DecordDecode"),
+    dict(type="Resize", scale=(-1, 224)),
+    dict(type="RandomResizedCrop"),
+    dict(type="Resize", scale=(224, 224), keep_ratio=False),
+    dict(type="Flip", flip_ratio=0.5),
+    dict(type="FormatShape", input_format="NCTHW"),
+    dict(type="PackActionInputs"),
+]  # type: ignore
+multiclass = True
+num_classes = 3
+test_mode = True

configs/models/videomaev2.py

@@ -0,0 +1,101 @@
+_base_ = ["../../mmaction2/configs/_base_/default_runtime.py"]
+
+custom_imports = dict(imports="datasets", allow_failed_imports=False)
+work_dir = "work_dirs/videomaev2"
+launcher = "none"
+
+
+# model settings
+model = dict(
+    type="Recognizer3D",
+    backbone=dict(
+        type="VisionTransformer",
+        img_size=224,
+        patch_size=16,
+        embed_dims=384,
+        depth=12,
+        num_heads=6,
+        mlp_ratio=4,
+        qkv_bias=True,
+        num_frames=16,
+        norm_cfg=dict(type="LN", eps=1e-6),
+    ),
+    cls_head=dict(
+        type="TimeSformerHead",
+        num_classes=3,
+        in_channels=384,
+        average_clips="prob",
+        multi_class=True,
+    ),
+    # TODO: update this to fit our dataset
+    data_preprocessor=dict(
+        type="ActionDataPreprocessor",
+        mean=[123.675, 116.28, 103.53],
+        std=[58.395, 57.12, 57.375],
+        format_shape="NCTHW",
+    ),
+)
+
+# dataset settings
+dataset_type = "HighQualityFallDataset"
+ann_file_train = "tests/test_data/test_annotation.csv"
+
+train_pipeline = [
+    dict(type="DecordInit"),
+    dict(type="ClipVideo"),
+    dict(type="SampleFrames", clip_len=16, frame_interval=4, num_clips=1),
+    dict(type="DecordDecode"),
+    dict(type="Resize", scale=(-1, 224)),
+    dict(type="RandomResizedCrop"),
+    dict(type="Resize", scale=(224, 224), keep_ratio=False),
+    dict(type="Flip", flip_ratio=0.5),
+    dict(type="FormatShape", input_format="NCTHW"),
+    dict(type="PackActionInputs"),
+]
+
+train_dataloader = dict(
+    batch_size=1,
+    num_workers=8,
+    persistent_workers=False,
+    sampler=dict(type="DefaultSampler", shuffle=False),
+    dataset=dict(
+        type=dataset_type,
+        sampling_strategy=dict(type="UniformSampling", clip_len=10),
+        label_strategy=dict(
+            type="ExistenceLabel",
+            label_description=dict(
+                names=["fall", "lying", "other"],
+                start_timestamp_names=["fall_start", "lying_start"],
+                end_timestamp_names=["fall_end", "lying_end"],
+                visible_names=["fall_visible", "lying_visible"],
+                other_class=2,
+            ),
+        ),
+        ann_file=ann_file_train,
+        pipeline=train_pipeline,
+        multi_class=True,
+        num_classes=3,
+    ),
+)
+
+train_cfg = dict(type="EpochBasedTrainLoop", max_epochs=5, val_interval=0)
+
+param_scheduler = dict(
+    type="MultiStepLR",  # Decays the learning rate once the number of epoch reaches one of the milestones
+    begin=0,  # Step at which to start updating the learning rate
+    end=100,  # Step at which to stop updating the learning rate
+    by_epoch=True,  # Whether the scheduled learning rate is updated by epochs
+    milestones=[40, 80],  # Steps to decay the learning rate
+    gamma=0.1,
+)
+
+optim_wrapper = dict(  # Config of optimizer wrapper
+    type="OptimWrapper",  # Name of optimizer wrapper, switch to AmpOptimWrapper to enable mixed precision training
+    optimizer=dict(  # Config of optimizer. Support all kinds of optimizers in PyTorch. Refer to https://pytorch.org/docs/stable/optim.html#algorithms
+        type="SGD",  # Name of optimizer
+        lr=0.01,  # Learning rate
+        momentum=0.9,  # Momentum factor
+        weight_decay=0.0001,
+    ),  # Weight decay
+    clip_grad=dict(max_norm=40, norm_type=2),
+)  # Config of gradient clip

containers/c3se_job_container.def

@@ -0,0 +1,15 @@
+Bootstrap: localimage
+From: /apps/containers/PyTorch/PyTorch-2.1.0-NGC-23.09.sif
+
+
+%files
+    containers/requirements.txt
+    /mimer/NOBACKUP/groups/naiss2023-22-1160/human-fall-detection/mmaction2
+
+%post
+    export DEBIAN_FRONTEND="noninteractive" && apt-get update -y && apt-get install -y python3-opencv
+    pip install -r containers/requirements.txt
+    mim install mmengine mmcv mmdet mmpose
+    cd /mimer/NOBACKUP/groups/naiss2023-22-1160/human-fall-detection/mmaction2 && pip install -v -e .
+    # For some reason there is a wrong version of opencv installed
+    pip uninstall -y opencv

containers/requirements.txt

@@ -0,0 +1,5 @@
+# PyTorch is preinstalled in the image
+openpyxl>=3.0
+openmim>=0.3
+ffmpeg-python>=0.2
+dvc[s3]

datasets/high_quality_fall_dataset.py

@@ -8,6 +8,7 @@
 
 from datasets.transforms.label_strategy import LabelStrategy
 from datasets.transforms.sampling_strategy import SamplingStrategy
+from registry import LABEL_STRATEGIES, SAMPLING_STRATEGIES
 
 
 @DATASETS.register_module()
@@ -64,8 +65,8 @@ class HighQualityFallDataset(BaseActionDataset):
     def __init__(
         self,
         ann_file: str,
-        sampling_strategy: SamplingStrategy,
-        label_strategy: LabelStrategy,
+        sampling_strategy: SamplingStrategy | dict,
+        label_strategy: LabelStrategy | dict,
         pipeline: List[Union[dict, Callable]],
         data_prefix: ConfigType = dict(video=""),
         multi_class: bool = False,
@@ -75,8 +76,16 @@ def __init__(
         test_mode: bool = False,
         **kwargs,
     ) -> None:
-        self.sampling_strategy = sampling_strategy
-        self.label_strategy = label_strategy
+        if isinstance(sampling_strategy, dict):
+            built_sampling_strategy = SAMPLING_STRATEGIES.build(sampling_strategy)  # type: SamplingStrategy
+        else:
+            built_sampling_strategy = sampling_strategy
+        self.sampling_strategy = built_sampling_strategy
+        if isinstance(label_strategy, dict):
+            built_label_strategy = LABEL_STRATEGIES.build(label_strategy)  # type: LabelStrategy
+        else:
+            built_label_strategy = label_strategy
+        self.label_strategy = built_label_strategy
         super().__init__(
             ann_file,
             pipeline=pipeline,
@@ -86,7 +95,6 @@ def __init__(
             start_index=start_index,
             modality=modality,
             test_mode=test_mode,
-            **kwargs,
         )
 
     def load_data_list(self) -> List[dict]:

datasets/transforms/label_strategy.py

@@ -1,9 +1,10 @@
 import abc
+from dataclasses import dataclass
 
 import pandas as pd
 
 from datasets.transforms.sampling_strategy import IntervalInSeconds
-from dataclasses import dataclass
+from registry import LABEL_STRATEGIES
 
 
 @dataclass
@@ -57,8 +58,11 @@ class LabelStrategy(abc.ABC):
     Args:
         label_description (LabelDescription): Description of the labels."""
 
-    def __init__(self, label_description: LabelDescription) -> None:
-        self.label_description = label_description
+    def __init__(self, label_description: LabelDescription | dict) -> None:
+        if isinstance(label_description, dict):
+            self.label_description = LabelDescription(**label_description)
+        else:
+            self.label_description = label_description
 
     @abc.abstractmethod
     def label(self, annotation: pd.Series, clip: IntervalInSeconds) -> list[int]:
@@ -75,6 +79,7 @@ def label(self, annotation: pd.Series, clip: IntervalInSeconds) -> list[int]:
         ...
 
 
+@LABEL_STRATEGIES.register_module()
 class ExistenceLabel(LabelStrategy):
     """Generates a label based on the existence of actions within the clip.
 
@@ -88,7 +93,7 @@ class ExistenceLabel(LabelStrategy):
 
     def __init__(
         self,
-        label_description: LabelDescription,
+        label_description: LabelDescription | dict,
         threshold: float = 0.0,
         absolute_threshold: bool = True,
     ) -> None:

datasets/transforms/sampling_strategy.py

@@ -3,6 +3,8 @@
 
 import pandas as pd
 
+from registry import SAMPLING_STRATEGIES
+
 IntervalInSeconds = Tuple[float, float]
 
 
@@ -24,6 +26,7 @@ def sample(self, annotation: pd.Series) -> List[IntervalInSeconds]:
         ...
 
 
+@SAMPLING_STRATEGIES.register_module()
 class UniformSampling(SamplingStrategy):
     """Samples uniformly from the untrimmed video.
 

environment.yml

@@ -27,3 +27,4 @@ dependencies:
       - jupyter>=1.0
       - ipympl>=0.9
       - pandas-stubs>=2.1
+      - ffmpeg-python>=0.2

job_scripts/toy_training.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+# apptainer exec \
+#     --env PYTHONPATH=$(pwd) \
+#     --env CUDA_VISIBLE_DEVICES=2,3 \
+#     containers/c3se_job_container.sif \
+#     python -m torch.distributed.launch --nproc_per_node=2 \
+#     mmaction2/tools/train.py \
+#     configs/models/videomaev2.py \
+#     --launcher pytorch
+
+apptainer exec \
+    --env PYTHONPATH=$(pwd) \
+    --env CUDA_VISIBLE_DEVICES=2,3 \
+    containers/c3se_job_container.sif \
+    python mmaction2/tools/train.py \
+    configs/models/videomaev2.py

registry.py

@@ -0,0 +1,10 @@
+from mmengine.registry import Registry
+
+
+SAMPLING_STRATEGIES = Registry(
+    "sampling strategy", locations=["datasets.transforms.sampling_strategy"], scope="."
+)
+
+LABEL_STRATEGIES = Registry(
+    "label strategy", locations=["datasets.transforms.label_strategy"], scope="."
+)