Signed commit

Deci-AI · Oct 18, 2023 · a6d4630 · a6d4630
1 parent b5eb12c
commit a6d4630
Show file tree

Hide file tree

Showing 262 changed files with 10,556 additions and 1,817 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -23,6 +23,10 @@ parameters:
     type: boolean
     description: If true will run sanity_test flow
     default: false
+  run_and_convert_notebooks_to_docs:
+    type: boolean
+    description: If true will run run_and_convert_notebooks_to_docs flow
+    default: false
   skip_md_files:
     type: boolean
     default: true
@@ -503,7 +507,7 @@ jobs:
             python3.8 -m pip install --upgrade setuptools pip wheel
             python3.8 -m pip install -r requirements.txt
             python3.8 -m pip install .
-            python3.8 -m pip install torch torchvision torchaudio   
+            python3.8 -m pip install torch torchvision torchaudio
             make sweeper_test
 
       - run:
@@ -547,7 +551,7 @@ jobs:
             python3.8 src/super_gradients/train_from_recipe.py --config-name=coco2017_pose_dekr_w32_no_dc experiment_name=shortened_coco2017_pose_dekr_w32_ap_test batch_size=4 val_batch_size=8 epochs=1 training_hyperparams.lr_warmup_steps=0 training_hyperparams.average_best_models=False training_hyperparams.max_train_batches=1000 training_hyperparams.max_valid_batches=100 multi_gpu=DDP num_gpus=4
             python3.8 src/super_gradients/train_from_recipe.py --config-name=cifar10_resnet experiment_name=shortened_cifar10_resnet_accuracy_test epochs=100 training_hyperparams.average_best_models=False multi_gpu=DDP num_gpus=4
             python3.8 src/super_gradients/examples/convert_recipe_example/convert_recipe_example.py --config-name=cifar10_conversion_params experiment_name=shortened_cifar10_resnet_accuracy_test
-            python3.8 src/super_gradients/train_from_recipe.py --config-name=coco2017_yolox experiment_name=shortened_coco2017_yolox_n_map_test architecture=yolox_n training_hyperparams.loss=yolox_fast_loss epochs=10 training_hyperparams.average_best_models=False multi_gpu=DDP num_gpus=4
+            python3.8 src/super_gradients/train_from_recipe.py --config-name=coco2017_yolox experiment_name=shortened_coco2017_yolox_n_map_test architecture=yolox_n training_hyperparams.loss=YoloXFastDetectionLoss epochs=10 training_hyperparams.average_best_models=False multi_gpu=DDP num_gpus=4
             python3.8 src/super_gradients/train_from_recipe.py --config-name=cityscapes_regseg48 experiment_name=shortened_cityscapes_regseg48_iou_test epochs=10 training_hyperparams.average_best_models=False multi_gpu=DDP num_gpus=4
             coverage run --source=super_gradients -m unittest tests/deci_core_recipe_test_suite_runner.py
 
@@ -561,6 +565,50 @@ jobs:
           event: fail
           template: basic_fail_1 # see https://github.com/CircleCI-Public/slack-orb/wiki#templates.
 
+  run_and_convert_notebooks_to_docs:
+    docker:
+      - image: 307629990626.dkr.ecr.us-east-1.amazonaws.com/deci/infra/circleci/runner/sg-gpu:<< pipeline.parameters.sg_docker_version >>
+    resource_class: deci-ai/sg-gpu-on-premise
+    parameters:
+      sg_existing_env_path:
+        type: string
+        default: "/env/persistent_env"
+      sg_new_env_name:
+        type: string
+        default: "${CIRCLE_BUILD_NUM}"
+      sg_new_env_python_version:
+        type: string
+        default: "python3.8"
+    steps:
+      - checkout
+      - run:
+          name: install requirements and execute notebooks
+          command: |
+            << parameters.sg_new_env_python_version >> -m venv << parameters.sg_new_env_name >>
+            source << parameters.sg_new_env_name >>/bin/activate
+            python3.8 -m pip install --upgrade setuptools pip wheel
+            python3.8 -m pip install -r requirements.txt
+            python3.8 -m pip install -r requirements.dev.txt
+            python3.8 -m pip install .
+            python3.8 -m pip install torch torchvision torchaudio
+            python3.8 -m pip install pytorch-quantization==2.1.2 --extra-index-url https://pypi.ngc.nvidia.com
+            python3.8 -m pip install onnx_graphsurgeon==0.3.27   --extra-index-url https://pypi.ngc.nvidia.com
+            python3.8 -m ipykernel install --name << parameters.sg_new_env_name >> --user
+            make check_notebooks_version_match
+            make run_and_convert_notebooks_to_docs
+
+      - run:
+          name: Remove new environment when failed
+          command: "rm -r << parameters.sg_new_env_name >>"
+          when: on_fail
+
+      - slack/notify:
+          channel: "sg-integration-tests"
+          event: fail
+          template: basic_fail_1 # see https://github.com/CircleCI-Public/slack-orb/wiki#templates.
+
+
+
   recipe_sanity_tests_classification_pt1:
     docker:
       - image: 307629990626.dkr.ecr.us-east-1.amazonaws.com/deci/infra/circleci/runner/sg-gpu:<< pipeline.parameters.sg_docker_version >>
@@ -822,28 +870,38 @@ jobs:
                 dockerfile: 'scripts/Dockerfile.branch'
 
 workflows:
-  sanity_tests:
-    when:  << pipeline.parameters.run_sanity_tests_flow >>
+#  sanity_tests:
+#    when:  << pipeline.parameters.run_sanity_tests_flow >>
+#    jobs:
+#      - hydra_sweeper_test:
+#          context:
+#            - slack
+#      - recipe_sanity_tests_classification_pt1:
+#          context:
+#            - slack
+#      - recipe_sanity_tests_classification_pt2:
+#          context:
+#            - slack
+#      - recipe_sanity_tests_segmentation:
+#          context:
+#            - slack
+#      - recipe_sanity_tests_detection:
+#          context:
+#            - slack
+#      - recipe_sanity_tests_pose_estimation:
+#          context:
+#            - slack
+#      - recipe_accuracy_tests:
+#          context:
+#            - slack
+#      - run_and_convert_notebooks_to_docs:
+#          context:
+#            - slack
+
+  run_and_convert_notebooks_to_docs:
+    when:  << pipeline.parameters.run_and_convert_notebooks_to_docs >>
     jobs:
-      - hydra_sweeper_test:
-          context:
-            - slack
-      - recipe_sanity_tests_classification_pt1:
-          context:
-            - slack
-      - recipe_sanity_tests_classification_pt2:
-          context:
-            - slack
-      - recipe_sanity_tests_segmentation:
-          context:
-            - slack
-      - recipe_sanity_tests_detection:
-          context:
-            - slack
-      - recipe_sanity_tests_pose_estimation:
-          context:
-            - slack
-      - recipe_accuracy_tests:
+      - run_and_convert_notebooks_to_docs:
           context:
             - slack
 
@@ -868,45 +926,50 @@ workflows:
             - deci-common/persist_version_info
             - login_to_codeartifact_release
           <<: *release_tag_filter
-      - hydra_sweeper_test:
-          context:
-            - slack
-          <<: *release_tag_filter
-      - recipe_sanity_tests_classification_pt1:
-          context:
-            - slack
-          <<: *release_tag_filter
-      - recipe_sanity_tests_classification_pt2:
-          context:
-            - slack
-          <<: *release_tag_filter
-      - recipe_sanity_tests_segmentation:
-          context:
-            - slack
-          <<: *release_tag_filter
-      - recipe_sanity_tests_detection:
-          context:
-            - slack
-          <<: *release_tag_filter
-      - recipe_sanity_tests_pose_estimation:
-          context:
-            - slack
-          <<: *release_tag_filter
-      - recipe_accuracy_tests:
-          context:
-            - slack
-          <<: *release_tag_filter
+#      - hydra_sweeper_test:
+#          context:
+#            - slack
+#          <<: *release_tag_filter
+#      - recipe_sanity_tests_classification_pt1:
+#          context:
+#            - slack
+#          <<: *release_tag_filter
+#      - recipe_sanity_tests_classification_pt2:
+#          context:
+#            - slack
+#          <<: *release_tag_filter
+#      - recipe_sanity_tests_segmentation:
+#          context:
+#            - slack
+#          <<: *release_tag_filter
+#      - recipe_sanity_tests_detection:
+#          context:
+#            - slack
+#          <<: *release_tag_filter
+#      - recipe_sanity_tests_pose_estimation:
+#          context:
+#            - slack
+#          <<: *release_tag_filter
+#      - recipe_accuracy_tests:
+#          context:
+#            - slack
+#          <<: *release_tag_filter
+#      - run_and_convert_notebooks_to_docs:
+#          context:
+#            - slack
+#          <<: *release_tag_filter
       - release_version:
           py_version: "3.8"
           requires:
             - "build3.8"
-            - hydra_sweeper_test
-            - recipe_accuracy_tests
-            - recipe_sanity_tests_classification_pt1
-            - recipe_sanity_tests_classification_pt2
-            - recipe_sanity_tests_segmentation
-            - recipe_sanity_tests_detection
-            - recipe_sanity_tests_pose_estimation
+#            - hydra_sweeper_test
+#            - recipe_accuracy_tests
+#            - recipe_sanity_tests_classification_pt1
+#            - recipe_sanity_tests_classification_pt2
+#            - recipe_sanity_tests_segmentation
+#            - recipe_sanity_tests_detection
+#            - recipe_sanity_tests_pose_estimation
+#            - run_and_convert_notebooks_to_docs
           <<: *release_tag_filter
       - deci-common/pip_upload_package_from_codeartifact_to_global_pypi:
           package_name: "super-gradients"

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -86,6 +86,23 @@ $ git config --global gpg.program /usr/local/bin/gpg
 $ git config --global commit.gpgsign true
 ```
 
+### I have created PR with unsigned commits. Do I have to start over?
+
+No. What you need to do is create another branch starting from master, then move your commits from PR branch to the 
+new branch and force push it to the remote under the old name. Let's say your PR branch named 'feature/my_awesome_pr'
+then you need to do the following:
+
+```bash
+git checkout master
+git checkout -b feature/my_awesome_pr_signed
+git merge --squash --no-commit feature/my_awesome_pr
+git commit -S -m "My signed commit"
+git push -f origin feature/my_awesome_pr_signed:feature/my_awesome_pr
+```
+
+Last command will overwrite your PR branch with the new signed commit containing all changes from the PR. 
+
+```bash
 
 ## Jupyter Notebooks Contribution
 

diff --git a/Makefile b/Makefile
@@ -10,7 +10,7 @@ yolo_nas_integration_tests:
 recipe_accuracy_tests:
 	python src/super_gradients/train_from_recipe.py --config-name=coco2017_pose_dekr_w32_no_dc experiment_name=shortened_coco2017_pose_dekr_w32_ap_test epochs=1 batch_size=4 val_batch_size=8 training_hyperparams.lr_warmup_steps=0 training_hyperparams.average_best_models=False training_hyperparams.max_train_batches=1000 training_hyperparams.max_valid_batches=100 multi_gpu=DDP num_gpus=4
 	python src/super_gradients/train_from_recipe.py --config-name=cifar10_resnet               experiment_name=shortened_cifar10_resnet_accuracy_test   epochs=100 training_hyperparams.average_best_models=False multi_gpu=DDP num_gpus=4
-	python src/super_gradients/train_from_recipe.py --config-name=coco2017_yolox               experiment_name=shortened_coco2017_yolox_n_map_test      epochs=10  architecture=yolox_n training_hyperparams.loss=yolox_fast_loss training_hyperparams.average_best_models=False multi_gpu=DDP num_gpus=4
+	python src/super_gradients/train_from_recipe.py --config-name=coco2017_yolox               experiment_name=shortened_coco2017_yolox_n_map_test      epochs=10  architecture=yolox_n training_hyperparams.loss=YoloXFastDetectionLoss training_hyperparams.average_best_models=False multi_gpu=DDP num_gpus=4
 	python src/super_gradients/train_from_recipe.py --config-name=cityscapes_regseg48          experiment_name=shortened_cityscapes_regseg48_iou_test   epochs=10 training_hyperparams.average_best_models=False multi_gpu=DDP num_gpus=4
 	python src/super_gradients/examples/convert_recipe_example/convert_recipe_example.py --config-name=cifar10_conversion_params experiment_name=shortened_cifar10_resnet_accuracy_test
 	coverage run --source=super_gradients -m unittest tests/deci_core_recipe_test_suite_runner.py
@@ -30,6 +30,14 @@ sweeper_test:
 	  exit 1; \
 	fi
 
+# Here you define a list of notebooks we want to execute and convert to markdown files
+# NOTEBOOKS = hellomake.ipynb hellofunc.ipynb helloclass.ipynb
+NOTEBOOKS = src/super_gradients/examples/model_export/models_export.ipynb
 
-examples_to_docs:
-	jupyter nbconvert --to markdown --output-dir="documentation/source/" --execute src/super_gradients/examples/model_export/models_export.ipynb
+# This Makefile target runs notebooks listed below and converts them to markdown files in documentation/source/
+run_and_convert_notebooks_to_docs: $(NOTEBOOKS)
+	jupyter nbconvert --to markdown --output-dir="documentation/source/" --execute $@
+
+# This Makefile target runs notebooks listed below and converts them to markdown files in documentation/source/
+check_notebooks_version_match: $(NOTEBOOKS)
+	python tests/verify_notebook_version.py $@
diff --git a/documentation/source/Checkpoints.md b/documentation/source/Checkpoints.md
@@ -79,8 +79,8 @@ model = models.get(model_name=Models.RESNET18, num_classes=10)
 
 train_params = {
     ...
-    "loss": "cross_entropy",
-    "criterion_params": {},
+    "loss": "LabelSmoothingCrossEntropyLoss",
+
     "save_ckpt_epoch_list": [10,15]
     ...
 }

diff --git a/documentation/source/Example_Classification.md b/documentation/source/Example_Classification.md
@@ -308,7 +308,6 @@ Output (Training parameters):
     'ckpt_name': 'ckpt_latest.pth',
     'clip_grad_norm': None,
     'cosine_final_lr_ratio': 0.01,
-    'criterion_params': {},
     'dataset_statistics': False,
     'ema': False,
     'ema_params': {'decay': 0.9999, 'decay_type': 'exp', 'beta': 15},
@@ -318,10 +317,10 @@ Output (Training parameters):
     'launch_tensorboard': False,
     'load_opt_params': True,
     'log_installed_packages': True,
-    'loss': 'cross_entropy',
+    'loss': "LabelSmoothingCrossEntropyLoss",
     'lr_cooldown_epochs': 0,
     'lr_decay_factor': 0.1,
-    'lr_mode': 'step',
+    'lr_mode': 'StepLRScheduler',
     'lr_schedule_function': None,
     'lr_updates': array([100, 150, 200]),
     'lr_warmup_epochs': 0,
@@ -355,7 +354,7 @@ Output (Training parameters):
     'train_metrics_list': ['Accuracy', 'Top5'],
     'valid_metrics_list': ['Accuracy', 'Top5'],
     'warmup_initial_lr': None,
-    'warmup_mode': 'linear_epoch_step',
+    'warmup_mode': 'LinearEpochLRWarmup',
     'zero_weight_decay_on_bias_and_bn': False
 }
 ```

diff --git a/documentation/source/Example_Training-an-external-model.md b/documentation/source/Example_Training-an-external-model.md
@@ -640,7 +640,7 @@ And lastly, we need to define the training hyperparameters:
 ```python
 train_params = {
     "max_epochs": 100,
-    "lr_mode": "cosine",
+    "lr_mode": "CosineLRScheduler",
     "initial_lr": 0.001,
     "optimizer": "Adam",
     "loss": CustomSegLoss(),