From 0a24740f359eae33ee57a5724efc47502fd79f77 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 10 Apr 2024 09:08:03 +0200 Subject: [PATCH 01/33] CM repo meta --- cmr.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 cmr.yaml diff --git a/cmr.yaml b/cmr.yaml new file mode 100644 index 0000000..3088cbe --- /dev/null +++ b/cmr.yaml @@ -0,0 +1,4 @@ +alias: mlcommons@cm4abtf +uid: 566d31eda11948a9 + +git: true From f92b9f464de89a38a4bde149290dede2d94c8631 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 10 Apr 2024 09:30:05 +0200 Subject: [PATCH 02/33] first commit --- .gitignore | 160 +++++++++++++ COPYRIGHT.txt | 1 + README-test-model.md | 117 ++++++++++ README.md | 1 + .../get-dataset-cognata-mlcommons/README.md | 187 +++++++++++++++ script/get-dataset-cognata-mlcommons/_cm.yaml | 65 ++++++ .../customize.py | 55 +++++ .../get-dataset-cognata-mlcommons/license.txt | 107 +++++++++ .../README-extra.md | 5 + .../get-ml-model-abtf-ssd-pytorch/README.md | 170 ++++++++++++++ script/get-ml-model-abtf-ssd-pytorch/_cm.yaml | 84 +++++++ .../customize.py | 36 +++ .../README.md | 218 ++++++++++++++++++ .../_cm.yaml | 89 +++++++ .../customize.py | 28 +++ .../test-ssd-resnet50-cognata-pytorch/run.bat | 6 + .../test-ssd-resnet50-cognata-pytorch/run.sh | 6 + 17 files changed, 1335 insertions(+) create mode 100644 .gitignore create mode 100644 COPYRIGHT.txt create mode 100644 README-test-model.md create mode 100644 README.md create mode 100644 script/get-dataset-cognata-mlcommons/README.md create mode 100644 script/get-dataset-cognata-mlcommons/_cm.yaml create mode 100644 script/get-dataset-cognata-mlcommons/customize.py create mode 100644 script/get-dataset-cognata-mlcommons/license.txt create mode 100644 script/get-ml-model-abtf-ssd-pytorch/README-extra.md create mode 100644 script/get-ml-model-abtf-ssd-pytorch/README.md create mode 100644 script/get-ml-model-abtf-ssd-pytorch/_cm.yaml create mode 100644 script/get-ml-model-abtf-ssd-pytorch/customize.py create mode 100644 script/test-ssd-resnet50-cognata-pytorch/README.md create mode 100644 script/test-ssd-resnet50-cognata-pytorch/_cm.yaml create mode 100644 script/test-ssd-resnet50-cognata-pytorch/customize.py create mode 100644 script/test-ssd-resnet50-cognata-pytorch/run.bat create mode 100644 script/test-ssd-resnet50-cognata-pytorch/run.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..68bc17f --- /dev/null +++ b/.gitignore @@ -0,0 +1,160 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt new file mode 100644 index 0000000..bb27d28 --- /dev/null +++ b/COPYRIGHT.txt @@ -0,0 +1 @@ +Copyright (c) 2024 MLCommons diff --git a/README-test-model.md b/README-test-model.md new file mode 100644 index 0000000..f9b7d14 --- /dev/null +++ b/README-test-model.md @@ -0,0 +1,117 @@ +# CM automation for ABTF-MLPerf + +*Testing ABTF SSD PyTorch model via the [MLCommons CM automation meta-framework](https://github.com/mlcommons/ck).* + +## Install CM + +Follow [this online guide](https://access.cknowledge.org/playground/?action=install) to install CM for your OS. + +## Install virtual environment + +We suggest to create a virtual environment to avoid messing up your Python installation: + +### Linux + +```bash +python3 -m venv ABTF +. ABTF/bin/activate ; export CM_REPOS=$PWD/ABTF/CM +``` +### Windows + +```bash +python -m venv ABTF +call ABTF\Scripts\activate.bat & set CM_REPOS=%CD%\ABTF\CM +``` + +## Install all CM automation recipes + +Pull [main MLOps automation recipes](https://access.cknowledge.org/playground/?action=scripts) from MLCommons: + +```bash +cm pull repo mlcommons@ck --checkout=dev +``` + +Pull this CM repository with automation recipes for the MLCommons-ABTF benchmark: + +```bash +cm pull repo cknowledge@cm4abtf +``` + +## Clean CM cache + +Clean CM cache if you want to start from scratch + +```bash +cm rm cache -f +``` + + + + + + + + + +Download private test image `0000008766.png` and model `baseline_8mp.pth` to your local directory. + + +Import `baseline_8mp.pth` to CM: +```bash +cmr "get ml-model abtf-ssd-pytorch _local.baseline_8mp.pth" +``` + +Get Git repo with ABTF SSD-ResNet50 PyTorch model: + +```bash +cmr "get git repo _repo.https://github.com/mlcommons/abtf-ssd-pytorch" --env.CM_GIT_BRANCH=cognata-cm --extra_cache_tags=abtf,ssd,pytorch,cm-model --env.CM_GIT_CHECKOUT_PATH_ENV_NAME=CM_ABTF_SSD_PYTORCH +``` + +Make test prediction: + +```bash +cmr "test abtf ssd-resnet50 cognata pytorch" --input=0000008766.png --output=0000008766_prediction_test.jpg --config=baseline_8MP +``` + +Export PyTorch model to ONNX: +```bash +cmr "test abtf ssd-resnet50 cognata pytorch" --input=0000008766.png --output=0000008766_prediction_test.jpg --config=baseline_8MP --export_model=baseline_8mp.onnx +``` + +Test exported ONNX model with LoadGen (performance): +```bash +cm run script "python app loadgen-generic _onnxruntime" --modelpath=baseline_8mp.onnx --samples=1 --quiet +``` + + +Test different versions of PyTorch +```bash +cmr "install python-venv" --name=abtf2 +cmr "test abtf ssd-resnet50 cognata pytorch" --adr.python.name=abtf2 --adr.torch.version=1.13.1 --adr.torchvision.version=0.14.1 --input=0000008766.png --output=0000008766_prediction_test.jpg --config=baseline_8MP +``` + +## TBD + +### Main features + +* Test PyTorch model with Python LoadGen +* Test PyTorch model with [C++ loadgen](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/app-mlperf-inference-mlcommons-cpp) +* Automate loading of Cognata dataset via CM +* Add Cognata dataset to loadgen +* Process PyTorch model with MLPerf inference infrastructure for SSD-ResNet50 +* Add support for MLCommons Croissant + +### Testing docker + +```bash +cm docker script --tags=test,abtf,ssd-pytorch,_cognata --docker_cm_repo=ctuning@mlcommons-ck --env.CM_GH_TOKEN={TOKEN} --input=road.jpg --output=road_ssd.jpg +``` + +```bash +cm docker script --tags=test,abtf,ssd-pytorch,_cognata --docker_cm_repo=ctuning@mlcommons-ck --docker_os=ubuntu --docker_os_version=23.04 --input=road.jpg --output=road_ssd.jpg +``` +TBD: pass file to CM docker: [meta](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/build-mlperf-inference-server-nvidia/_cm.yaml#L197). + +## CM automation developers + +* [Grigori Fursin](https://cKnowledge.org/gfursin) (MLCommons Task Force on Automation and Reproducibility) diff --git a/README.md b/README.md new file mode 100644 index 0000000..b609e1c --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# Collective Mind interface and automation for ABTF diff --git a/script/get-dataset-cognata-mlcommons/README.md b/script/get-dataset-cognata-mlcommons/README.md new file mode 100644 index 0000000..606d817 --- /dev/null +++ b/script/get-dataset-cognata-mlcommons/README.md @@ -0,0 +1,187 @@ +
+Click here to see the table of contents. + +* [About](#about) +* [Summary](#summary) +* [Reuse this script in your project](#reuse-this-script-in-your-project) + * [ Install CM automation language](#install-cm-automation-language) + * [ Check CM script flags](#check-cm-script-flags) + * [ Run this script from command line](#run-this-script-from-command-line) + * [ Run this script from Python](#run-this-script-from-python) + * [ Run this script via GUI](#run-this-script-via-gui) + * [ Run this script via Docker (beta)](#run-this-script-via-docker-(beta)) +* [Customization](#customization) + * [ Variations](#variations) + * [ Default environment](#default-environment) +* [Script workflow, dependencies and native scripts](#script-workflow-dependencies-and-native-scripts) +* [Script output](#script-output) +* [New environment keys (filter)](#new-environment-keys-(filter)) +* [New environment keys auto-detected from customize](#new-environment-keys-auto-detected-from-customize) +* [Maintainers](#maintainers) + +
+ +*Note that this README is automatically generated - don't edit!* + +### About + +#### Summary + +* Category: *AI/ML datasets.* +* CM GitHub repository: *[mlcommons@ck](https://github.com/mlcommons/ck/tree/master/cm-mlops)* +* GitHub directory for this script: *[GitHub](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-dataset-cognata)* +* CM meta description for this script: *[_cm.json](_cm.json)* +* CM "database" tags to find this script: *get,dataset,cognata,object-detection,original* +* Output cached? *True* +___ +### Reuse this script in your project + +#### Install CM automation language + +* [Installation guide](https://github.com/mlcommons/ck/blob/master/docs/installation.md) +* [CM intro](https://doi.org/10.5281/zenodo.8105339) + +#### Pull CM repository with this automation + +```cm pull repo mlcommons@ck``` + + +#### Run this script from command line + +1. `cm run script --tags=get,dataset,cognata,object-detection,original[,variations] ` + +2. `cmr "get dataset cognata object-detection original[ variations]" ` + +* `variations` can be seen [here](#variations) + +#### Run this script from Python + +
+Click here to expand this section. + +```python + +import cmind + +r = cmind.access({'action':'run' + 'automation':'script', + 'tags':'get,dataset,cognata,object-detection,original' + 'out':'con', + ... + (other input keys for this script) + ... + }) + +if r['return']>0: + print (r['error']) + +``` + +
+ + +#### Run this script via GUI + +```cmr "cm gui" --script="get,dataset,cognata,object-detection,original"``` + +Use this [online GUI](https://cKnowledge.org/cm-gui/?tags=get,dataset,cognata,object-detection,original) to generate CM CMD. + +#### Run this script via Docker (beta) + +`cm docker script "get dataset cognata object-detection original[ variations]" ` + +___ +### Customization + + +#### Variations + + * Group "**dataset-type**" +
+ Click here to expand this section. + + * `_calibration` + - Workflow: + * **`_validation`** (default) + - Environment variables: + - *CM_DATASET_CALIBRATION*: `no` + - Workflow: + +
+ + + * Group "**size**" +
+ Click here to expand this section. + + * **`_50`** (default) + - Environment variables: + - *CM_DATASET_SIZE*: `50` + - Workflow: + * `_500` + - Environment variables: + - *CM_DATASET_SIZE*: `500` + - Workflow: + * `_full` + - Environment variables: + - *CM_DATASET_SIZE*: `` + - Workflow: + * `_size.#` + - Environment variables: + - *CM_DATASET_SIZE*: `#` + - Workflow: + +
+ + +#### Default variations + +`_50,_validation` +#### Default environment + +
+Click here to expand this section. + +These keys can be updated via `--env.KEY=VALUE` or `env` dictionary in `@input.json` or using script flags. + +* CM_DATASET_CALIBRATION: `no` + +
+ +___ +### Script workflow, dependencies and native scripts + +
+Click here to expand this section. + + 1. ***Read "deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-dataset-cognata/_cm.json)*** + * get,python3 + * CM names: `--adr.['python', 'python3']...` + - CM script: [get-python3](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-python3) + 1. ***Run "preprocess" function from [customize.py](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-dataset-cognata/customize.py)*** + 1. Read "prehook_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-dataset-cognata/_cm.json) + 1. ***Run native script if exists*** + 1. Read "posthook_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-dataset-cognata/_cm.json) + 1. ***Run "postrocess" function from [customize.py](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-dataset-cognata/customize.py)*** + 1. Read "post_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-dataset-cognata/_cm.json) +
+ +___ +### Script output +`cmr "get dataset cognata object-detection original[,variations]" -j` +#### New environment keys (filter) + +* `CM_CALIBRATION_DATASET_PATH` +* `CM_DATASET_ANNOTATIONS_DIR_PATH` +* `CM_DATASET_ANNOTATIONS_FILE_PATH` +* `CM_DATASET_PATH` +* `CM_DATASET_PATH_ROOT` +#### New environment keys auto-detected from customize + +* `CM_CALIBRATION_DATASET_PATH` +* `CM_DATASET_PATH` +* `CM_DATASET_PATH_ROOT` +___ +### Maintainers + +* [Open MLCommons taskforce on automation and reproducibility](https://github.com/mlcommons/ck/blob/master/docs/taskforce.md) \ No newline at end of file diff --git a/script/get-dataset-cognata-mlcommons/_cm.yaml b/script/get-dataset-cognata-mlcommons/_cm.yaml new file mode 100644 index 0000000..b2bf5c4 --- /dev/null +++ b/script/get-dataset-cognata-mlcommons/_cm.yaml @@ -0,0 +1,65 @@ +alias: get-dataset-cognata-mlcommons +uid: 6e2b5db76833416d + +automation_alias: script +automation_uid: 5b4e0237da074764 + +private: true + +cache: true + +category: AI/ML datasets +category_sort: 8500 + +default_env: + CM_DATASET_CALIBRATION: 'no' + +deps: +- names: + - python + - python3 + tags: get,python3 + +env: + CM_DATASET: COGNATA + +new_env_keys: +- CM_DATASET_PATH +- CM_DATASET_PATH_ROOT +- CM_DATASET_ANNOTATIONS_DIR_PATH +- CM_DATASET_ANNOTATIONS_FILE_PATH +- CM_CALIBRATION_DATASET_PATH + +tags: +- get +- dataset +- mlcommons-cognata +- object-detection +- original + +variations: + '50': + default: true + env: + CM_DATASET_SIZE: '50' + group: size + '500': + env: + CM_DATASET_SIZE: '500' + group: size + calibration: + env: {} + group: dataset-type + full: + env: + CM_DATASET_SIZE: '' + group: size + size.#: + env: + CM_DATASET_SIZE: '#' + group: size + validation: + default: true + env: + CM_DATASET_CALIBRATION: 'no' + group: dataset-type diff --git a/script/get-dataset-cognata-mlcommons/customize.py b/script/get-dataset-cognata-mlcommons/customize.py new file mode 100644 index 0000000..6c55731 --- /dev/null +++ b/script/get-dataset-cognata-mlcommons/customize.py @@ -0,0 +1,55 @@ +from cmind import utils +import os +import shutil +import sys + + +def ask_user(question, default="yes"): + valid = ["yes", "y", "no", "n"] + if default is None: + prompt = " [y/n] " + elif default == "yes": + prompt = " [Y/n] " + elif default == "no": + prompt = " [y/N] " + else: + raise ValueError("invalid default answer: '%s'" % default) + + while True: + sys.stdout.write(question + prompt) + choice = input().lower() + if default is not None and choice == "": + return default.startswith('y') + elif choice in valid: + return choice.startswith('y') + else: + print("Please input y/n\n") + +def preprocess(i): + + env = i['env'] + + script_path = i['run_script_input']['path'] + + with open(os.path.join(script_path, "license.txt"), "r") as f: + print(f.read()) + + response = ask_user("Do you accept?") + + print(response) + + + return {'return': 0} + +def postprocess(i): + env = i['env'] + return {'return': -1} #todo + if env.get('CM_DATASET_CALIBRATION','') == "no": + env['CM_DATASET_PATH_ROOT'] = os.path.join(os.getcwd(), 'install') + env['CM_DATASET_PATH'] = os.path.join(os.getcwd(), 'install', 'validation', 'data') + env['CM_DATASET_CAPTIONS_DIR_PATH'] = os.path.join(os.getcwd(), 'install', 'captions') + env['CM_DATASET_LATENTS_DIR_PATH'] = os.path.join(os.getcwd(), 'install', 'latents') + else: + env['CM_CALIBRATION_DATASET_PATH'] = os.path.join(os.getcwd(), 'install', 'calibration', 'data') + + return {'return': 0} diff --git a/script/get-dataset-cognata-mlcommons/license.txt b/script/get-dataset-cognata-mlcommons/license.txt new file mode 100644 index 0000000..f46fa42 --- /dev/null +++ b/script/get-dataset-cognata-mlcommons/license.txt @@ -0,0 +1,107 @@ +DATA END USER LICENSE AGREEMENT + +This Data End User License Agreement (“Agreement”) is by and between MLCommons, a Delaware non- +profit corporation (“MLC”) and the undersigned MLCommons Member that is downloading or using the + +MLCommons Cognata Dataset made available by MLCommons (“Licensee”). This Agreement will be +effective upon the date last signed, as reflected in the signature block below. +1. Background. MLC, an open engineering consortium, has licensed a dataset known as MLCommons +Cognata Dataset (the “Data”) which it wishes to make available to its Members in accordance with +the terms set forth in this Agreement. The term Data refers collectively to the MLCommons Cognata +datasets and materials delivered by MLC as a whole, to subsets of such datasets and materials, as +well as to the separate images and annotations comprising such datasets and materials. MLC agrees +that, to the best of its knowledge, the Data will not include any data that is “personal data” or +“personally identifiable” according to any applicable law or regulation. +2. Data License. Subject to the License Restrictions set forth in Section 4 below, MLC grants to Licensee +a non-exclusive, non-transferable, non-sublicenseable worldwide license during the term of the +Agreement to download, install, copy, modify, distribute, publicly display, prepare derivative works, +and otherwise use the Data for the purpose of benchmarking. The rights and license granted under +this Section 2.1 shall be referred to hereinafter as the “License.” +3. Derivative Work, Applications and Output. The License includes the right to create derivative +works of the Data. Any use of the Data incorporated in these derivative works will be governed by +the provisions of this Agreement, however, MLC and its licensors will not be responsible for any +adaptations, revisions, transformations, alterations, or any other modifications of the Data. The +License includes the right to incorporate and use the Data in applications (the “Applications”), +provided that all Data incorporated or used by these Applications is and will remain governed by the +provisions of this Agreement. The term “Output” refers to any analysis, results and calculations +arrived at from the use of the Data, and to any changes to the models and networks that will be +trained using the Data. +4. License Restrictions. +a. The License will only remain in effect so long as Licensee remains an MLC member in good +standing. +b. Licensee may not provide the Data as a whole, or on a standalone basis, to third parties but may +distribute the Applications and Output in the course of Licensee’s ordinary course business. This +includes the right to distribute a subset of the Data in Licensee’s Applications to run relevant +benchmarks generated from the Output. +c. While the License includes the right to use the Data in commercial activity such as +benchmarking, marketing, customer demonstrations, bidding and requests for proposals or +information, it excludes the right to use the Data directly in connection with models used in or +items in production, including training production models. +d. Licensee acknowledges that MLC’s data licensor, Cognata Ltd, is the owner of all right, title and +interest in the Data. It will acknowledge and identify “Cognata” as a data provider in connection +with any distribution, display or publication of the Data, Applications, or Output. +e. Licensee will ensure that its use of the Data will be done in accordance with all applicable law. + + +5. Term. The License will continue in effect until terminated as set forth below: +a. Licensee may terminate the License by informing MLC that all Data, Output and Applications +that incorporate the Data in Licensee’s possession or control have been deleted. +b. MLC may terminate the License upon providing Licensee with at least thirty (30) days prior +notice of termination sent to the email address of the primary contact associated with +Licensee’s Membership in MLC. +c. The License will terminate immediately and automatically without any notice if Licensee ceases +to be an MLC Member or if Licensee violates any of the terms and conditions of this Agreement. +Upon termination of this Agreement, Licensee must delete all copies of the Data and Output in its +possession and cease distributing any Data, Output or Applications that incorporate the Data. +Termination will not limit any of MLC’s rights or remedies at law or in equity. Sections 8 through 11 +of this Agreement, inclusive, shall survive such termination. +6. Storage. For the duration of this Agreement, Licensee may download and store local copies of the +Data on systems under its control. +7. License. This Agreement provides certain license rights to use of the Data but does not constitute a +sale of the Data. MLC’s licensor will continue to own all right, title and interest in the Data. +8. Indemnification. Licensee agrees to indemnify and hold MLC and its licensors, and their respective +officers, directors and agents harmless from any claims, expenses, or liabilities caused by Licensee’s +use of the Data or Licensee’s publication or use of Output; except that Licensee shall have no liability +or obligation to indemnify MLC or its licensor from or against any claims, expenses, or liabilities +resulting from (a) third party claims that the Data infringes, misappropriation, or otherwise violates +third party intellectual property or other rights or (b) MLC’s negligent acts or omissions or willful +misconduct. Licensee will promptly inform MLC in writing of any claim by a third party that use of +the Data in accordance with this Agreement violates the proprietary rights of such claimant +(including any threat, warning or notice prior to such claim in which the claimant alleges that the +Data infringes his/her/its intellectual property rights), which notice will include copies of all available +material related to the claim and will indicate the estimated amount claimed by the claimant, if any. +9. Disclaimers & Limited Liability. +EXCEPT AS EXPLICITLY SET FORTH IN THIS AGREEMENT, THE DATA IS PROVIDED “AS IS,” WITHOUT +WARRANTY OR REPRESENTATION OF ANY KIND WHETHER EXPRESS, IMPLIED, STATUTORY, OR +OTHER. THIS INCLUDES, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, +ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT KNOWN OR +DISCOVERABLE. LICENSEES ARE RESPONSIBLE FOR ANY AND ALL CONCLUSIONS, DECISIONS AND/OR +ACTIONS TAKEN BY THEM IN RELIANCE ON THE DATA. LICENSEE ASSUMES ALL RESPONSIBILITIES +FOR ANY USE OF THE OUTPUT. IN NO EVENT WILL COGNATA LTD, MLC’S DATA LICENSOR, +RESPONSIBLE TO LICENSEE IN ANY WAY FOR THE DATA. +EXCEPT FOR MATTERS FOR WHICH LIABILITY CANNOT BE EXCLUDED OR LIMITED UNDER APPLICABLE +LAW: (A) IN NO EVENT SHALL EITHER PARTY BE LIABLE FOR INCIDENTAL, INDIRECT, SPECIAL, +EXEMPLARY, PUNITIVE AND/OR CONSEQUENTIAL DAMAGES (HOWEVER ARISING) AND/OR FOR +LOSS OF PROFIT, LOSS OF USE, LOSS OF DATA, LOSS OF REVENUES, LOSS OF SAVINGS, BUSINESS +INTERRUPTION, OR LOSS OF REPUTATION ARISING FROM AND/OR IN CONNECTION WITH THIS +AGREEMENT, AND/OR THE USE OF THE DATA AND (B) THE MAXIMUM AGGREGATE LIABILITY OF + + +EITHER PARTY FOR ANY AND ALL DAMAGES AND LOSSES ARISING FROM AND/OR IN CONNECTION +WITH THIS AGREEMENT, AND/OR THE USE OF THE DATA SHALL NOT EXCEED THE AMOUNTS PAID +FOR THE APPLICABLE DATA. +LICENSEE AGREES THAT IT IS NOT ENTITLED TO RECOVER ANY OTHER DAMAGES OR LOSSES EVEN IF +THE ABOVE DAMAGES REMEDY DOESN'T FULLY COMPENSATE LICENSEE FOR ANY DAMAGES OR +LOSSES OR FAILS OF ITS ESSENTIAL PURPOSE AND EVEN IF MLC KNEW OR SHOULD HAVE KNOWN +ABOUT THE POSSIBILITY OF THE DAMAGES OR LOSSES. THE ABOVE LIMITATIONS AND EXCLUSIONS +OF LIABILITY SHALL APPLY TO ALL CAUSES OF ACTION AND REGARDLESS OF THE THEORY OF +LIABILITY (CONTRACT, TORT, EQUITY, BREACH OF STATUTORY DUTY, STRICT LIABILITY OR +OTHERWISE) AND EVEN IF SUCH DAMAGES OR LOSSES WERE OR COULD HAVE BEEN FORESEEABLE. +10. Amendment. MLC may update this Agreement upon 30 days’ written notice to Licensee. +11. Dispute resolution. This Agreement will be governed by the laws of the State of Delaware, without +reference to conflict of laws principles. If unable to resolve a dispute amicably, the parties agree that +disputes arising out of or related to this Agreement will be subject to the exclusive jurisdiction of the +state and federal courts located in San Francisco, California. +IN WITNESS WHEREOF, the undersigned parties have executed this Data End User License Agreement as +of the dates set forth below their signatures. diff --git a/script/get-ml-model-abtf-ssd-pytorch/README-extra.md b/script/get-ml-model-abtf-ssd-pytorch/README-extra.md new file mode 100644 index 0000000..c1d4093 --- /dev/null +++ b/script/get-ml-model-abtf-ssd-pytorch/README-extra.md @@ -0,0 +1,5 @@ +# Example to import local model + +```bash +cmr "get ml-model abtf-ssd-pytorch _local.test_8mp.pth" +``` diff --git a/script/get-ml-model-abtf-ssd-pytorch/README.md b/script/get-ml-model-abtf-ssd-pytorch/README.md new file mode 100644 index 0000000..7fc218f --- /dev/null +++ b/script/get-ml-model-abtf-ssd-pytorch/README.md @@ -0,0 +1,170 @@ +Automatically generated README for this automation recipe: **get-ml-model-abtf-ssd-pytorch** + +Category: **AI/ML models** + +License: **Apache 2.0** + +Maintainers: [Public MLCommons Task Force on Automation and Reproducibility](https://github.com/mlcommons/ck/blob/master/docs/taskforce.md) + +--- +*[ [Online info and GUI to run this CM script](https://access.cknowledge.org/playground/?action=scripts&name=get-ml-model-abtf-ssd-pytorch,59cfc2a22f5d4f46) ] [ [Notes from the authors, contributors and users](README-extra.md) ]* + +--- +#### Summary + +* CM GitHub repository: *[mlcommons@ck](https://github.com/mlcommons/ck/tree/dev/cm-mlops)* +* GitHub directory for this script: *[GitHub](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-ml-model-abtf-ssd-pytorch)* +* CM meta description for this script: *[_cm.json](_cm.json)* +* All CM tags to find and reuse this script (see in above meta description): *get,ml-model,abtf-ssd-pytorch* +* Output cached? *True* +* See [pipeline of dependencies](#dependencies-on-other-cm-scripts) on other CM scripts + + +--- +### Reuse this script in your project + +#### Install MLCommons CM automation meta-framework + +* [Install CM](https://access.cknowledge.org/playground/?action=install) +* [CM Getting Started Guide](https://github.com/mlcommons/ck/blob/master/docs/getting-started.md) + +#### Pull CM repository with this automation recipe (CM script) + +```cm pull repo mlcommons@ck``` + +#### Print CM help from the command line + +````cmr "get ml-model abtf-ssd-pytorch" --help```` + +#### Customize and run this script from the command line with different variations and flags + +`cm run script --tags=get,ml-model,abtf-ssd-pytorch` + +`cm run script --tags=get,ml-model,abtf-ssd-pytorch[,variations] ` + +*or* + +`cmr "get ml-model abtf-ssd-pytorch"` + +`cmr "get ml-model abtf-ssd-pytorch [variations]" ` + + +* *See the list of `variations` [here](#variations) and check the [Gettings Started Guide](https://github.com/mlcommons/ck/blob/dev/docs/getting-started.md) for more details.* + +#### Run this script from Python + +
+Click here to expand this section. + +```python + +import cmind + +r = cmind.access({'action':'run' + 'automation':'script', + 'tags':'get,ml-model,abtf-ssd-pytorch' + 'out':'con', + ... + (other input keys for this script) + ... + }) + +if r['return']>0: + print (r['error']) + +``` + +
+ + +#### Run this script via GUI + +```cmr "cm gui" --script="get,ml-model,abtf-ssd-pytorch"``` + +Use this [online GUI](https://cKnowledge.org/cm-gui/?tags=get,ml-model,abtf-ssd-pytorch) to generate CM CMD. + +#### Run this script via Docker (beta) + +`cm docker script "get ml-model abtf-ssd-pytorch[variations]" ` + +___ +### Customization + + +#### Variations + + * Group "**epoch**" +
+ Click here to expand this section. + + * `_e01` + - Environment variables: + - *CM_ML_MODEL_CHECKSUM*: `31d177228308bbe43917c912b01c2d67` + - *CM_ML_MODEL_FILENAME*: `SSD_e1.pth` + - *CM_ML_MODEL_URL*: `https://www.dropbox.com/scl/fi/7nqt5z8gplgeaveo933eo/SSD_e1.pth?rlkey=7lyb4qs2hzg491bfprwcuvx54&dl=0` + - *CM_ML_MODEL*: `abtf-ssd-pytorch` + - *CM_ML_MODEL_DATASET*: `coco` + - *CM_ML_MODEL_IMAGE_HEIGHT*: `300` + - *CM_ML_MODEL_IMAGE_WIDTH*: `300` + - Workflow: + * **`_e65`** (default) + - Environment variables: + - *CM_ML_MODEL_CHECKSUM*: `f769eb0321ac7fc1c16f982db6131d2f` + - *CM_ML_MODEL_FILENAME*: `SSD_e65.pth` + - *CM_ML_MODEL_URL*: `https://www.dropbox.com/scl/fi/wkegl2qxvm8cefbqq00o3/SSD_e65.pth?rlkey=ez26jafjdcly665npl6pdqxl8&dl=0` + - *CM_ML_MODEL*: `abtf-ssd-pytorch` + - *CM_ML_MODEL_DATASET*: `coco` + - *CM_ML_MODEL_IMAGE_HEIGHT*: `300` + - *CM_ML_MODEL_IMAGE_WIDTH*: `300` + - Workflow: + * `_local.#` + - Environment variables: + - *CM_ML_MODEL_FILENAME*: `#` + - *CM_ML_MODEL_LOCAL*: `yes` + - *CM_SKIP_DOWNLOAD*: `yes` + - Workflow: + +
+ + +#### Default variations + +`_e65` +#### Default environment + +
+Click here to expand this section. + +These keys can be updated via `--env.KEY=VALUE` or `env` dictionary in `@input.json` or using script flags. + + +
+ +___ +### Dependencies on other CM scripts + + + 1. ***Read "deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-ml-model-abtf-ssd-pytorch/_cm.json)*** + * detect,os + - CM script: [detect-os](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/detect-os) + * download,file,_wget + * `if (CM_SKIP_DOWNLOAD != yes)` + * CM names: `--adr.['get-ml-model']...` + - CM script: [download-file](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/download-file) + 1. ***Run "preprocess" function from [customize.py](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-ml-model-abtf-ssd-pytorch/customize.py)*** + 1. Read "prehook_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-ml-model-abtf-ssd-pytorch/_cm.json) + 1. ***Run native script if exists*** + 1. Read "posthook_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-ml-model-abtf-ssd-pytorch/_cm.json) + 1. ***Run "postrocess" function from [customize.py](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-ml-model-abtf-ssd-pytorch/customize.py)*** + 1. Read "post_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-ml-model-abtf-ssd-pytorch/_cm.json) + +___ +### Script output +`cmr "get ml-model abtf-ssd-pytorch [,variations]" -j` +#### New environment keys (filter) + +* `CM_ML_MODEL_*` +#### New environment keys auto-detected from customize + +* `CM_ML_MODEL_FILE` +* `CM_ML_MODEL_FILE_WITH_PATH` \ No newline at end of file diff --git a/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml b/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml new file mode 100644 index 0000000..a30d93d --- /dev/null +++ b/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml @@ -0,0 +1,84 @@ +alias: get-ml-model-abtf-ssd-pytorch +uid: 59cfc2a22f5d4f46 + +automation_alias: script +automation_uid: 5b4e0237da074764 + +cache: true + +category: AI/ML models + +private: true + +deps: + +- tags: detect,os + +- tags: get,git,repo,_repo.https://github.com/mlcommons/abtf-ssd-pytorch + names: + - abtf-ssd-pytorch-git-repo + env: + CM_GIT_CHECKOUT_PATH_ENV_NAME: CM_ABTF_SSD_PYTORCH + CM_GIT_BRANCH: cognata-cm + extra_cache_tags: abtf,ssd,pytorch,cm-model + + +- tags: download,file,_wget + env: + CM_DOWNLOAD_CHECKSUM: <<>> + CM_DOWNLOAD_FINAL_ENV_NAME: CM_ML_MODEL_FILE_WITH_PATH + CM_VERIFY_SSL: 'no' + force_cache: true + names: + - get-ml-model-weights + skip_if_env: + CM_SKIP_DOWNLOAD: + - 'yes' + update_tags_from_env_with_prefix: + _url.: + - CM_ML_MODEL_URL + + + +new_env_keys: +- CM_ML_MODEL_* + +print_env_at_the_end: + CM_ML_MODEL_FILE_WITH_PATH: Path to the ML model weights + CM_ML_MODEL_CODE_WITH_PATH: Path to the ML model code + +tags: +- get +- ml-model +- abtf-ssd-pytorch + +variations: + e01: + env: + CM_ML_MODEL: abtf-ssd-pytorch + CM_ML_MODEL_CHECKSUM: 31d177228308bbe43917c912b01c2d67 + CM_ML_MODEL_DATASET: coco + CM_ML_MODEL_FILENAME: SSD_e1.pth + CM_ML_MODEL_IMAGE_HEIGHT: '300' + CM_ML_MODEL_IMAGE_WIDTH: '300' + CM_ML_MODEL_URL: https://www.dropbox.com/scl/fi/7nqt5z8gplgeaveo933eo/SSD_e1.pth?rlkey=7lyb4qs2hzg491bfprwcuvx54&dl=0 + group: epoch + + e65: + default: true + env: + CM_ML_MODEL: abtf-ssd-pytorch + CM_ML_MODEL_CHECKSUM: f769eb0321ac7fc1c16f982db6131d2f + CM_ML_MODEL_DATASET: coco + CM_ML_MODEL_FILENAME: SSD_e65.pth + CM_ML_MODEL_IMAGE_HEIGHT: '300' + CM_ML_MODEL_IMAGE_WIDTH: '300' + CM_ML_MODEL_URL: https://www.dropbox.com/scl/fi/wkegl2qxvm8cefbqq00o3/SSD_e65.pth?rlkey=ez26jafjdcly665npl6pdqxl8&dl=0 + group: epoch + + local.#: + env: + CM_ML_MODEL_FILENAME: '#' + CM_ML_MODEL_LOCAL: 'yes' + CM_SKIP_DOWNLOAD: 'yes' + group: epoch diff --git a/script/get-ml-model-abtf-ssd-pytorch/customize.py b/script/get-ml-model-abtf-ssd-pytorch/customize.py new file mode 100644 index 0000000..f8d8fb7 --- /dev/null +++ b/script/get-ml-model-abtf-ssd-pytorch/customize.py @@ -0,0 +1,36 @@ +from cmind import utils +import os + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + if env.get('CM_ML_MODEL_LOCAL', '') == 'yes': + ml_model = env.get('CM_ML_MODEL_FILENAME', '') + if ml_model == '': + return {'return':1, 'error':'_local.{model name.pth} is not specified'} + + if not os.path.isabs(ml_model): + ml_model = os.path.join(env.get('CM_TMP_CURRENT_PATH',''), ml_model) + + if not os.path.isfile(ml_model): + return {'return':1, 'error':'ML model {} is not found'.format(ml_model)} + + env['CM_ML_MODEL_FILE_WITH_PATH'] = ml_model + + return {'return':0} + +def postprocess(i): + + env = i['env'] + + env['CM_ML_MODEL_FILE'] = os.path.basename(env['CM_ML_MODEL_FILE_WITH_PATH']) + + env['CM_ML_MODEL_CODE_WITH_PATH'] = env['CM_ABTF_SSD_PYTORCH'] + + env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_ML_MODEL_FILE_WITH_PATH'] + + return {'return':0} + diff --git a/script/test-ssd-resnet50-cognata-pytorch/README.md b/script/test-ssd-resnet50-cognata-pytorch/README.md new file mode 100644 index 0000000..cadb4c2 --- /dev/null +++ b/script/test-ssd-resnet50-cognata-pytorch/README.md @@ -0,0 +1,218 @@ +Automatically generated README for this automation recipe: **test-abtf-ssd-pytorch** + +Category: **Tests** + +License: **Apache 2.0** + +Maintainers: [Public MLCommons Task Force on Automation and Reproducibility](https://github.com/mlcommons/ck/blob/master/docs/taskforce.md) + +--- +*[ [Online info and GUI to run this CM script](https://access.cknowledge.org/playground/?action=scripts&name=test-abtf-ssd-pytorch,91bfc4333b054c21) ]* + +--- +#### Summary + +* CM GitHub repository: *[mlcommons@ck](https://github.com/mlcommons/ck/tree/dev/cm-mlops)* +* GitHub directory for this script: *[GitHub](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/test-abtf-ssd-pytorch)* +* CM meta description for this script: *[_cm.yaml](_cm.yaml)* +* All CM tags to find and reuse this script (see in above meta description): *test,abtf,ssd,pytorch,ssd-pytorch* +* Output cached? *False* +* See [pipeline of dependencies](#dependencies-on-other-cm-scripts) on other CM scripts + + +--- +### Reuse this script in your project + +#### Install MLCommons CM automation meta-framework + +* [Install CM](https://access.cknowledge.org/playground/?action=install) +* [CM Getting Started Guide](https://github.com/mlcommons/ck/blob/master/docs/getting-started.md) + +#### Pull CM repository with this automation recipe (CM script) + +```cm pull repo mlcommons@ck``` + +#### Print CM help from the command line + +````cmr "test abtf ssd pytorch ssd-pytorch" --help```` + +#### Customize and run this script from the command line with different variations and flags + +`cm run script --tags=test,abtf,ssd,pytorch,ssd-pytorch` + +`cm run script --tags=test,abtf,ssd,pytorch,ssd-pytorch[,variations] [--input_flags]` + +*or* + +`cmr "test abtf ssd pytorch ssd-pytorch"` + +`cmr "test abtf ssd pytorch ssd-pytorch [variations]" [--input_flags]` + + +* *See the list of `variations` [here](#variations) and check the [Gettings Started Guide](https://github.com/mlcommons/ck/blob/dev/docs/getting-started.md) for more details.* + + +#### Input Flags + +* --**input**=input image (png) +* --**output**=output image (png) +* --**export_model**=ONNX model name to be exported from PyTorch + +**Above CLI flags can be used in the Python CM API as follows:** + +```python +r=cm.access({... , "input":...} +``` +#### Run this script from Python + +
+Click here to expand this section. + +```python + +import cmind + +r = cmind.access({'action':'run' + 'automation':'script', + 'tags':'test,abtf,ssd,pytorch,ssd-pytorch' + 'out':'con', + ... + (other input keys for this script) + ... + }) + +if r['return']>0: + print (r['error']) + +``` + +
+ + +#### Run this script via GUI + +```cmr "cm gui" --script="test,abtf,ssd,pytorch,ssd-pytorch"``` + +Use this [online GUI](https://cKnowledge.org/cm-gui/?tags=test,abtf,ssd,pytorch,ssd-pytorch) to generate CM CMD. + +#### Run this script via Docker (beta) + +`cm docker script "test abtf ssd pytorch ssd-pytorch[variations]" [--input_flags]` + +___ +### Customization + + +#### Variations + + * Group "**dataset**" +
+ Click here to expand this section. + + * **`_coco`** (default) + - Environment variables: + - *CM_ABTF_DATASET*: `coco` + - *CM_ABTF_SSD_PYTORCH_BRANCH*: `main` + - Workflow: + * `_cognata` + - Environment variables: + - *CM_ABTF_DATASET*: `Cognata` + - *CM_ABTF_SSD_PYTORCH_BRANCH*: `cognata` + - *CM_ABTF_ML_MODEL_CONFIG*: `baseline_8MP` + - Workflow: + +
+ + + * Group "**device**" +
+ Click here to expand this section. + + * **`_cpu`** (default) + - Environment variables: + - *CM_DEVICE*: `cpu` + - Workflow: + * `_cuda` + - Environment variables: + - *CM_DEVICE*: `cuda` + - Workflow: + +
+ + +#### Default variations + +`_coco,_cpu` + +#### Script flags mapped to environment +
+Click here to expand this section. + +* `--export_model=value` → `CM_ABTF_EXPORT_MODEL_TO_ONNX=value` +* `--input=value` → `CM_INPUT_IMAGE=value` +* `--output=value` → `CM_OUTPUT_IMAGE=value` + +**Above CLI flags can be used in the Python CM API as follows:** + +```python +r=cm.access({... , "export_model":...} +``` + +
+ +#### Default environment + +
+Click here to expand this section. + +These keys can be updated via `--env.KEY=VALUE` or `env` dictionary in `@input.json` or using script flags. + + +
+ +___ +### Dependencies on other CM scripts + + + 1. ***Read "deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/test-abtf-ssd-pytorch/_cm.yaml)*** + * detect,os + - CM script: [detect-os](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/detect-os) + * get,python3 + * CM names: `--adr.['python', 'python3']...` + - CM script: [get-python3](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-python3) + * get,generic-python-lib,_numpy + - CM script: [get-generic-python-lib](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-generic-python-lib) + * get,generic-python-lib,_package.Pillow + - CM script: [get-generic-python-lib](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-generic-python-lib) + * get,generic-python-lib,_onnx + - CM script: [get-generic-python-lib](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-generic-python-lib) + * get,generic-python-lib,_torch + * CM names: `--adr.['torch']...` + - CM script: [get-generic-python-lib](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-generic-python-lib) + * get,generic-python-lib,_torchvision + - CM script: [get-generic-python-lib](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-generic-python-lib) + * get,generic-python-lib,_opencv-python + - CM script: [get-generic-python-lib](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-generic-python-lib) + * get,ml-model,abtf-ssd-pytorch + * CM names: `--adr.['ml-model']...` + - CM script: [get-ml-model-abtf-ssd-pytorch](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-ml-model-abtf-ssd-pytorch) + * get,git,repo,_repo.https://github.com/mlcommons/abtf-ssd-pytorch + * CM names: `--adr.['abtf-ssd-pytorch-git-repo']...` + - CM script: [get-git-repo](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-git-repo) + 1. ***Run "preprocess" function from [customize.py](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/test-abtf-ssd-pytorch/customize.py)*** + 1. Read "prehook_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/test-abtf-ssd-pytorch/_cm.yaml) + 1. ***Run native script if exists*** + * [run-coco.bat](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/test-abtf-ssd-pytorch/run-coco.bat) + * [run-coco.sh](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/test-abtf-ssd-pytorch/run-coco.sh) + * [run-cognata.bat](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/test-abtf-ssd-pytorch/run-cognata.bat) + * [run-cognata.sh](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/test-abtf-ssd-pytorch/run-cognata.sh) + 1. Read "posthook_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/test-abtf-ssd-pytorch/_cm.yaml) + 1. ***Run "postrocess" function from [customize.py](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/test-abtf-ssd-pytorch/customize.py)*** + 1. Read "post_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/test-abtf-ssd-pytorch/_cm.yaml) + +___ +### Script output +`cmr "test abtf ssd pytorch ssd-pytorch [,variations]" [--input_flags] -j` +#### New environment keys (filter) + +#### New environment keys auto-detected from customize diff --git a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml new file mode 100644 index 0000000..d9d4ad3 --- /dev/null +++ b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml @@ -0,0 +1,89 @@ +alias: test-ssd-resnet50-cognata-pytorch +uid: 8a063cb8582a4a96 + +automation_alias: script +automation_uid: 5b4e0237da074764 + +private: true + +category: Tests + +tags: +- test +- abtf +- ssd-resnet50 +- cognata +- pytorch + +input_mapping: + input: CM_INPUT_IMAGE + output: CM_OUTPUT_IMAGE + export_model: CM_ABTF_EXPORT_MODEL_TO_ONNX + config: CM_ABTF_ML_MODEL_CONFIG + dataset: CM_ABTF_DATASET + num_classes: CM_ABTF_NUM_CLASSES + +default_env: + CM_ABTF_ML_MODEL_CONFIG: baseline_8MP + CM_ABTF_DATASET: Cognata + +deps: + +- tags: detect,os + +- tags: get,python3 + names: + - python + - python3 + +- tags: get,generic-python-lib,_numpy +- tags: get,generic-python-lib,_package.Pillow +- tags: get,generic-python-lib,_opencv-python +- tags: get,generic-python-lib,_onnx +- tags: get,generic-python-lib,_torch + names: + - torch +- tags: get,generic-python-lib,_torchvision + + +- tags: get,ml-model,abtf-ssd-pytorch + names: + - ml-model + + + + +variations: + cpu: + group: device + default: True + env: + CM_DEVICE: cpu + + cuda: + group: + device + env: + CM_DEVICE: cuda + +input_description: + input: + desc: input image (png) + output: + desc: output image (png) + export_model: + desc: ONNX model name to be exported from PyTorch + num-classes: + desc: change number of classes (13) + +docker: + skip_run_cmd: 'no' + all_gpus: 'yes' + input_paths: + - input + - output + - export_model + skip_input_for_fake_run: + - input + - output + - export_model diff --git a/script/test-ssd-resnet50-cognata-pytorch/customize.py b/script/test-ssd-resnet50-cognata-pytorch/customize.py new file mode 100644 index 0000000..e752428 --- /dev/null +++ b/script/test-ssd-resnet50-cognata-pytorch/customize.py @@ -0,0 +1,28 @@ +from cmind import utils +import os + +def preprocess(i): + os_info = i['os_info'] + + env = i['env'] + + print ('') + print ('Current directory: {}'.format(os.getcwd())) + + print ('') + + extra = '' + if env.get('CM_ABTF_NUM_CLASSES', '')!='': + extra +=' --num-classes '+str(env['CM_ABTF_NUM_CLASSES']) + + if extra!='': + print ('') + print ('Extra command line: {}'.format(extra)) + + env['CM_ABTF_EXTRA_CMD'] = extra + + return {'return':0} + +def postprocess(i): + + return {'return':0} diff --git a/script/test-ssd-resnet50-cognata-pytorch/run.bat b/script/test-ssd-resnet50-cognata-pytorch/run.bat new file mode 100644 index 0000000..09eec39 --- /dev/null +++ b/script/test-ssd-resnet50-cognata-pytorch/run.bat @@ -0,0 +1,6 @@ +@echo off + +echo ======================================================= + +%CM_PYTHON_BIN_WITH_PATH% %CM_ML_MODEL_CODE_WITH_PATH%\test_image.py --pretrained-model "%CM_ML_MODEL_FILE_WITH_PATH%" --dataset %CM_ABTF_DATASET% --config %CM_ABTF_ML_MODEL_CONFIG% --input %CM_INPUT_IMAGE% --output %CM_OUTPUT_IMAGE% %CM_ABTF_EXTRA_CMD% +IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% diff --git a/script/test-ssd-resnet50-cognata-pytorch/run.sh b/script/test-ssd-resnet50-cognata-pytorch/run.sh new file mode 100644 index 0000000..64f8026 --- /dev/null +++ b/script/test-ssd-resnet50-cognata-pytorch/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +echo "=======================================================" + +${CM_PYTHON_BIN_WITH_PATH} ${CM_ML_MODEL_CODE_WITH_PATH}/test_image.py --pretrained-model "${CM_ML_MODEL_FILE_WITH_PATH}" --dataset ${CM_ABTF_DATASET} --config ${CM_ABTF_ML_MODEL_CONFIG} --input ${CM_INPUT_IMAGE} --output ${CM_OUTPUT_IMAGE} ${CM_ABTF_EXTRA_CMD} +test $? -eq 0 || exit $? From 2dc19546d269da747799e03eedbe255f575de288 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 10 Apr 2024 13:15:51 +0200 Subject: [PATCH 03/33] various fixes --- script/get-ml-model-abtf-ssd-pytorch/_cm.yaml | 1 + .../_cm.yaml | 37 +++++++++++++++++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml b/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml index a30d93d..914d928 100644 --- a/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml +++ b/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml @@ -51,6 +51,7 @@ tags: - get - ml-model - abtf-ssd-pytorch +- cmc variations: e01: diff --git a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml index d9d4ad3..c16c04b 100644 --- a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml +++ b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml @@ -40,10 +40,40 @@ deps: - tags: get,generic-python-lib,_package.Pillow - tags: get,generic-python-lib,_opencv-python - tags: get,generic-python-lib,_onnx -- tags: get,generic-python-lib,_torch + + +######################################################################## +# Install PyTorch for CPU or CUDA + +- enable_if_env: + CM_DEVICE: + - cpu + tags: get,generic-python-lib,_torch + names: + - torch + +- enable_if_env: + CM_DEVICE: + - cpu + tags: get,generic-python-lib,_torchvision + names: + - torchvision + +# CUDA + +- enable_if_env: + CM_DEVICE: + - cuda + tags: get,generic-python-lib,_torch_cuda names: - torch -- tags: get,generic-python-lib,_torchvision + +- enable_if_env: + CM_DEVICE: + - cuda + tags: get,generic-python-lib,_torchvision_cuda + names: + - torchvision - tags: get,ml-model,abtf-ssd-pytorch @@ -74,7 +104,8 @@ input_description: export_model: desc: ONNX model name to be exported from PyTorch num-classes: - desc: change number of classes (13) + desc: change number of classes (default = 13) + docker: skip_run_cmd: 'no' From 2d9542050eeb0da088648d40cef38d975d320f39 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 10 Apr 2024 15:33:49 +0200 Subject: [PATCH 04/33] clean up --- script/get-ml-model-abtf-ssd-pytorch/_cm.yaml | 55 ++++++++++++++----- .../customize.py | 6 ++ 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml b/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml index 914d928..914d338 100644 --- a/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml +++ b/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml @@ -10,16 +10,38 @@ category: AI/ML models private: true + +tags: +- get +- ml-model +- abtf-ssd-pytorch +- cmc + + +input_mapping: + model_code_git_url: CM_ABTF_MODEL_CODE_GIT_URL + model_code_git_branch: CM_ABTF_MODEL_CODE_GIT_BRANCH + + +default_env: + CM_ABTF_MODEL_CODE_GIT_URL: https://github.com/mlcommons/abtf-ssd-pytorch + CM_ABTF_MODEL_CODE_GIT_BRANCH: cognata-cm + + deps: - tags: detect,os -- tags: get,git,repo,_repo.https://github.com/mlcommons/abtf-ssd-pytorch +- tags: get,git,repo,_repo.<<>> names: - abtf-ssd-pytorch-git-repo + - abtf-ml-model-code-git-repo + skip_if_env: + CM_SKIP_MODEL_CODE_DOWNLOAD: + - 'yes' env: CM_GIT_CHECKOUT_PATH_ENV_NAME: CM_ABTF_SSD_PYTORCH - CM_GIT_BRANCH: cognata-cm + CM_GIT_BRANCH: <<>> extra_cache_tags: abtf,ssd,pytorch,cm-model @@ -30,9 +52,10 @@ deps: CM_VERIFY_SSL: 'no' force_cache: true names: - - get-ml-model-weights + - abtf-ml-model-weights + - abtf-ml-model-weights-download skip_if_env: - CM_SKIP_DOWNLOAD: + CM_SKIP_MODEL_WEIGHTS_DOWNLOAD: - 'yes' update_tags_from_env_with_prefix: _url.: @@ -47,11 +70,6 @@ print_env_at_the_end: CM_ML_MODEL_FILE_WITH_PATH: Path to the ML model weights CM_ML_MODEL_CODE_WITH_PATH: Path to the ML model code -tags: -- get -- ml-model -- abtf-ssd-pytorch -- cmc variations: e01: @@ -63,10 +81,9 @@ variations: CM_ML_MODEL_IMAGE_HEIGHT: '300' CM_ML_MODEL_IMAGE_WIDTH: '300' CM_ML_MODEL_URL: https://www.dropbox.com/scl/fi/7nqt5z8gplgeaveo933eo/SSD_e1.pth?rlkey=7lyb4qs2hzg491bfprwcuvx54&dl=0 - group: epoch + group: model-weights e65: - default: true env: CM_ML_MODEL: abtf-ssd-pytorch CM_ML_MODEL_CHECKSUM: f769eb0321ac7fc1c16f982db6131d2f @@ -75,11 +92,21 @@ variations: CM_ML_MODEL_IMAGE_HEIGHT: '300' CM_ML_MODEL_IMAGE_WIDTH: '300' CM_ML_MODEL_URL: https://www.dropbox.com/scl/fi/wkegl2qxvm8cefbqq00o3/SSD_e65.pth?rlkey=ez26jafjdcly665npl6pdqxl8&dl=0 - group: epoch + group: model-weights local.#: env: CM_ML_MODEL_FILENAME: '#' CM_ML_MODEL_LOCAL: 'yes' - CM_SKIP_DOWNLOAD: 'yes' - group: epoch + CM_SKIP_MODEL_WEIGHTS_DOWNLOAD: 'yes' + group: model-weights + + skip_weights: + default: true + env: + CM_SKIP_MODEL_WEIGHTS_DOWNLOAD: 'yes' + group: model-weights + + skip_code: + env: + CM_SKIP_MODEL_CODE_DOWNLOAD: 'yes' diff --git a/script/get-ml-model-abtf-ssd-pytorch/customize.py b/script/get-ml-model-abtf-ssd-pytorch/customize.py index f8d8fb7..44142ab 100644 --- a/script/get-ml-model-abtf-ssd-pytorch/customize.py +++ b/script/get-ml-model-abtf-ssd-pytorch/customize.py @@ -26,8 +26,14 @@ def postprocess(i): env = i['env'] + if env.get('CM_ML_MODEL_FILE_WITH_PATH', '') == '': + env['CM_ML_MODEL_FILE_WITH_PATH'] = 'model-weights-skipped' + env['CM_ML_MODEL_FILE'] = os.path.basename(env['CM_ML_MODEL_FILE_WITH_PATH']) + if env.get('CM_ABTF_SSD_PYTORCH', '') == '': + env['CM_ABTF_SSD_PYTORCH'] = 'model-code-skipped' + env['CM_ML_MODEL_CODE_WITH_PATH'] = env['CM_ABTF_SSD_PYTORCH'] env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_ML_MODEL_FILE_WITH_PATH'] From bdc534cee4c1243f81b91068040bcdea719d0798 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 10 Apr 2024 16:05:20 +0200 Subject: [PATCH 05/33] added README --- README-test-model.md | 117 ------------------------------------------ README.md | 119 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 117 deletions(-) delete mode 100644 README-test-model.md diff --git a/README-test-model.md b/README-test-model.md deleted file mode 100644 index f9b7d14..0000000 --- a/README-test-model.md +++ /dev/null @@ -1,117 +0,0 @@ -# CM automation for ABTF-MLPerf - -*Testing ABTF SSD PyTorch model via the [MLCommons CM automation meta-framework](https://github.com/mlcommons/ck).* - -## Install CM - -Follow [this online guide](https://access.cknowledge.org/playground/?action=install) to install CM for your OS. - -## Install virtual environment - -We suggest to create a virtual environment to avoid messing up your Python installation: - -### Linux - -```bash -python3 -m venv ABTF -. ABTF/bin/activate ; export CM_REPOS=$PWD/ABTF/CM -``` -### Windows - -```bash -python -m venv ABTF -call ABTF\Scripts\activate.bat & set CM_REPOS=%CD%\ABTF\CM -``` - -## Install all CM automation recipes - -Pull [main MLOps automation recipes](https://access.cknowledge.org/playground/?action=scripts) from MLCommons: - -```bash -cm pull repo mlcommons@ck --checkout=dev -``` - -Pull this CM repository with automation recipes for the MLCommons-ABTF benchmark: - -```bash -cm pull repo cknowledge@cm4abtf -``` - -## Clean CM cache - -Clean CM cache if you want to start from scratch - -```bash -cm rm cache -f -``` - - - - - - - - - -Download private test image `0000008766.png` and model `baseline_8mp.pth` to your local directory. - - -Import `baseline_8mp.pth` to CM: -```bash -cmr "get ml-model abtf-ssd-pytorch _local.baseline_8mp.pth" -``` - -Get Git repo with ABTF SSD-ResNet50 PyTorch model: - -```bash -cmr "get git repo _repo.https://github.com/mlcommons/abtf-ssd-pytorch" --env.CM_GIT_BRANCH=cognata-cm --extra_cache_tags=abtf,ssd,pytorch,cm-model --env.CM_GIT_CHECKOUT_PATH_ENV_NAME=CM_ABTF_SSD_PYTORCH -``` - -Make test prediction: - -```bash -cmr "test abtf ssd-resnet50 cognata pytorch" --input=0000008766.png --output=0000008766_prediction_test.jpg --config=baseline_8MP -``` - -Export PyTorch model to ONNX: -```bash -cmr "test abtf ssd-resnet50 cognata pytorch" --input=0000008766.png --output=0000008766_prediction_test.jpg --config=baseline_8MP --export_model=baseline_8mp.onnx -``` - -Test exported ONNX model with LoadGen (performance): -```bash -cm run script "python app loadgen-generic _onnxruntime" --modelpath=baseline_8mp.onnx --samples=1 --quiet -``` - - -Test different versions of PyTorch -```bash -cmr "install python-venv" --name=abtf2 -cmr "test abtf ssd-resnet50 cognata pytorch" --adr.python.name=abtf2 --adr.torch.version=1.13.1 --adr.torchvision.version=0.14.1 --input=0000008766.png --output=0000008766_prediction_test.jpg --config=baseline_8MP -``` - -## TBD - -### Main features - -* Test PyTorch model with Python LoadGen -* Test PyTorch model with [C++ loadgen](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/app-mlperf-inference-mlcommons-cpp) -* Automate loading of Cognata dataset via CM -* Add Cognata dataset to loadgen -* Process PyTorch model with MLPerf inference infrastructure for SSD-ResNet50 -* Add support for MLCommons Croissant - -### Testing docker - -```bash -cm docker script --tags=test,abtf,ssd-pytorch,_cognata --docker_cm_repo=ctuning@mlcommons-ck --env.CM_GH_TOKEN={TOKEN} --input=road.jpg --output=road_ssd.jpg -``` - -```bash -cm docker script --tags=test,abtf,ssd-pytorch,_cognata --docker_cm_repo=ctuning@mlcommons-ck --docker_os=ubuntu --docker_os_version=23.04 --input=road.jpg --output=road_ssd.jpg -``` -TBD: pass file to CM docker: [meta](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/build-mlperf-inference-server-nvidia/_cm.yaml#L197). - -## CM automation developers - -* [Grigori Fursin](https://cKnowledge.org/gfursin) (MLCommons Task Force on Automation and Reproducibility) diff --git a/README.md b/README.md index b609e1c..4916adb 100644 --- a/README.md +++ b/README.md @@ -1 +1,120 @@ # Collective Mind interface and automation for ABTF + +This repository contains [MLCommons CM](https://github.com/mlcommons/ck) automation recipes +to make it easier to prepare and benchmark different versions of ABTF models +(public or private) with MLPerf loadgen across different software and hardware. + + +## Install MLCommons CM automation framework + +Follow [this online guide](https://access.cknowledge.org/playground/?action=install) +to install CM for your OS with minimal dependencies. + + +## Install virtual environment + +We suggest to create a virtual environment to avoid messing up your Python installation. +All CM repositories, artifacts and cache will be resided inside this virtual environment +and can be removed at any time without influencing your own environment! + +### Linux + +```bash +python3 -m venv ABTF +. ABTF/bin/activate ; export CM_REPOS=$PWD/ABTF/CM +``` +### Windows + +```bash +python -m venv ABTF +call ABTF\Scripts\activate.bat & set CM_REPOS=%CD%\ABTF\CM +``` + +## Install CM automations + +### CM for MLOps + +```bash +cm pull repo mlcommons@ck --checkout=dev +``` + +### CM for ABTF + +```bash +cm pull repo cknowledge@cm4abtf --checkout=dev +``` + + +## Download test ABTF model and image + +```bash +cmr "download file _wget" --url="https://www.dropbox.com/scl/fi/od48qvnbqyfuy1z3aas84/baseline_8mp_ss_scales_ep15.pth?rlkey=d6ybe7g09g21pondmbd3pivzk&dl=0" --verify=no --md5sum=c36cb56b5f6bf8edbe64f9914506e09d +cmr "download file _wget" --url="https://www.dropbox.com/scl/fi/0n7rmxxwqvg04sxk7bbum/0000008766.png?rlkey=mhmr3ztrlsqk8oa67qtxoowuh&dl=0" --verify=no --md5sum=903306a7c8bfbe6c1ca68fad6e34fe52 +``` + +Check CM cache: +```bash +cm show cache +``` + +## Prepare workflow to benchmark ABTF model on host CPU + +### Build MLPerf loadgen + +```bash +cmr "get mlperf inference loadgen _copy" --version=main +``` + +### Install or detect PyTorch + +```bash +cmr "get generic-python-lib _torch" +cmr "get generic-python-lib _torchvision" +``` + +### Install ABTF model PyTorch code + +```bash +cmr "get ml-model abtf-ssd-pytorch _local.baseline_8mp_ss_scales_ep15.pth" +``` + + +## Test Model with a test image + +```bash +cmr "test abtf ssd-resnet50 cognata pytorch" --input=0000008766.png --output=0000008766_prediction_test.jpg --config=baseline_8MP_ss_scales --num-classes=13 +``` + +## Benchmark model with MLPerf loadgen + +```bash +cmr "generic loadgen python _pytorch _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cpu.pickle --modelpath=baseline_8mp.pth --modelcfg.num_classes=13 --modelcfg.config=baseline_8MP_ss_scales +``` + + +## Benchmarking other models + +Other ways to download public or private model code and weights: +```bash +cmr "get ml-model abtf-ssd-pytorch _skip_weights" --adr.abtf-ml-model-code-git-repo.env.CM_ABTF_MODEL_CODE_GIT_URL=https://github.com/mlcommons/abtf-ssd-pytorch +cmr "get ml-model abtf-ssd-pytorch _skip_weights" --model_code_git_url=https://github.com/mlcommons/abtf-ssd-pytorch --model_code_git_branch=cognata-cm +cmr "get ml-model abtf-ssd-pytorch _skip_weights _skip_code" +``` + +Other ways to run local (private) model: + +``` +cmr "generic loadgen python _pytorch _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cpu.pickle ^ + --modelpath=baseline_8mp.pth ^ + --modelcfg.num_classes=13 ^ + --modelcodepath="my-model-code" ^ + --modelcfg.config=baseline_8MP_ss_scales +``` + + + + +## Feedback + +Join MLCommons discord or get in touch with developer: gfursin@cknowledge.org + From 75c4d70a8e2066662292f19c4cd2e7085bfa08b0 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 10 Apr 2024 16:16:46 +0200 Subject: [PATCH 06/33] Updated docs --- README-cuda.md | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 18 ++++++++---- 2 files changed, 88 insertions(+), 5 deletions(-) create mode 100644 README-cuda.md diff --git a/README-cuda.md b/README-cuda.md new file mode 100644 index 0000000..f2ba17a --- /dev/null +++ b/README-cuda.md @@ -0,0 +1,75 @@ +[ [Back to the main page](README.md) ] + +*Tested with PyTorch 2.2.2 and CUDA 11.8 and 12.1* + +## Prepare workflow to benchmark ABTF model on CUDA-based device + +We expect that you already have CUDA driver installed + +### Detect or install CUDA toolkit and libraries + +``bash +cmr "get cuda _toolkit _cudnn" +cmr "get cuda-devices" +``` + +### Install or detect PyTorch and PyTorchVision + +#### CUDA 11.8 + +```bash +cmr "get generic-python-lib _torch_cuda" --extra-index-url=https://download.pytorch.org/whl/cu118 --force-install +cmr "get generic-python-lib _torchvision_cuda" --extra-index-url=https://download.pytorch.org/whl/cu118 --force-install +``` + +#### CUDA 12.1 + +```bash +cmr "get generic-python-lib _torch_cuda" --extra-index-url=https://download.pytorch.org/whl/cu121 --force-install +cmr "get generic-python-lib _torchvision_cuda" --extra-index-url=https://download.pytorch.org/whl/cu121 --force-install +``` + + + + + +## Test Model with a test image + +```bash +cmr "test abtf ssd-resnet50 cognata pytorch _cuda" --input=0000008766.png --output=0000008766_prediction_test.jpg --config=baseline_8MP_ss_scales --num-classes=13 +``` + +## Benchmark model with MLPerf loadgen + +```bash +cmr "generic loadgen python _pytorch _cuda _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cuda.pickle --modelpath=baseline_8mp.pth --modelcfg.num_classes=13 --modelcfg.config=baseline_8MP_ss_scales +``` + + +## Benchmarking other models + +Other ways to download public or private model code and weights: +```bash +cmr "get ml-model abtf-ssd-pytorch _skip_weights" --adr.abtf-ml-model-code-git-repo.env.CM_ABTF_MODEL_CODE_GIT_URL=https://github.com/mlcommons/abtf-ssd-pytorch +cmr "get ml-model abtf-ssd-pytorch _skip_weights" --model_code_git_url=https://github.com/mlcommons/abtf-ssd-pytorch --model_code_git_branch=cognata-cm +cmr "get ml-model abtf-ssd-pytorch _skip_weights _skip_code" +``` + +Other ways to run local (private) model: + +``` +cmr "generic loadgen python _pytorch _cuda _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cpu.pickle ^ + --modelpath=baseline_8mp.pth ^ + --modelcfg.num_classes=13 ^ + --modelcodepath="my-model-code" ^ + --modelcfg.config=baseline_8MP_ss_scales +``` + + + + + +## Feedback + +Join MLCommons discord or get in touch with developer: gfursin@cknowledge.org + diff --git a/README.md b/README.md index 4916adb..61feda6 100644 --- a/README.md +++ b/README.md @@ -52,13 +52,25 @@ cmr "download file _wget" --url="https://www.dropbox.com/scl/fi/od48qvnbqyfuy1z3 cmr "download file _wget" --url="https://www.dropbox.com/scl/fi/0n7rmxxwqvg04sxk7bbum/0000008766.png?rlkey=mhmr3ztrlsqk8oa67qtxoowuh&dl=0" --verify=no --md5sum=903306a7c8bfbe6c1ca68fad6e34fe52 ``` +### Install ABTF model PyTorch code + +```bash +cmr "get ml-model abtf-ssd-pytorch _local.baseline_8mp_ss_scales_ep15.pth" +``` + + Check CM cache: ```bash cm show cache ``` + ## Prepare workflow to benchmark ABTF model on host CPU +Next commands prepare environment to benchmark host CPU. +Check these docs to benchmark other devices: +* [CUDA-based device](README-cuda.md) + ### Build MLPerf loadgen ```bash @@ -72,11 +84,6 @@ cmr "get generic-python-lib _torch" cmr "get generic-python-lib _torchvision" ``` -### Install ABTF model PyTorch code - -```bash -cmr "get ml-model abtf-ssd-pytorch _local.baseline_8mp_ss_scales_ep15.pth" -``` ## Test Model with a test image @@ -114,6 +121,7 @@ cmr "generic loadgen python _pytorch _custom _cmc" --samples=5 --modelsamplepath + ## Feedback Join MLCommons discord or get in touch with developer: gfursin@cknowledge.org From 2c2a0db2034bbfdf93ebb32f43066fb92d781faa Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 10 Apr 2024 16:38:53 +0200 Subject: [PATCH 07/33] fixed typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 61feda6..8df369a 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ cm pull repo mlcommons@ck --checkout=dev ### CM for ABTF ```bash -cm pull repo cknowledge@cm4abtf --checkout=dev +cm pull repo mlcommons@cm4abtf --checkout=dev ``` From 4401e1e87e83c55d9feac6dd4554880cdb561f98 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 10 Apr 2024 16:46:22 +0200 Subject: [PATCH 08/33] fixed typos in READMEs --- README-cuda.md | 12 +++++++----- README.md | 12 +++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/README-cuda.md b/README-cuda.md index f2ba17a..3634938 100644 --- a/README-cuda.md +++ b/README-cuda.md @@ -42,7 +42,7 @@ cmr "test abtf ssd-resnet50 cognata pytorch _cuda" --input=0000008766.png --outp ## Benchmark model with MLPerf loadgen ```bash -cmr "generic loadgen python _pytorch _cuda _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cuda.pickle --modelpath=baseline_8mp.pth --modelcfg.num_classes=13 --modelcfg.config=baseline_8MP_ss_scales +cmr "generic loadgen python _pytorch _cuda _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cuda.pickle --modelpath=baseline_8mp_ss_scales_ep15.pth --modelcfg.num_classes=13 --modelcfg.config=baseline_8MP_ss_scales ``` @@ -57,11 +57,13 @@ cmr "get ml-model abtf-ssd-pytorch _skip_weights _skip_code" Other ways to run local (private) model: +You can first copy ABTF model code from GitHub to your local directory `my-model-code`. + ``` -cmr "generic loadgen python _pytorch _cuda _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cpu.pickle ^ - --modelpath=baseline_8mp.pth ^ - --modelcfg.num_classes=13 ^ - --modelcodepath="my-model-code" ^ +cmr "generic loadgen python _pytorch _cuda _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cpu.pickle \ + --modelpath=baseline_8mp_ss_scales_ep15.pth \ + --modelcfg.num_classes=13 \ + --modelcodepath="my-model-code" \ --modelcfg.config=baseline_8MP_ss_scales ``` diff --git a/README.md b/README.md index 8df369a..06f58fc 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ cmr "test abtf ssd-resnet50 cognata pytorch" --input=0000008766.png --output=000 ## Benchmark model with MLPerf loadgen ```bash -cmr "generic loadgen python _pytorch _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cpu.pickle --modelpath=baseline_8mp.pth --modelcfg.num_classes=13 --modelcfg.config=baseline_8MP_ss_scales +cmr "generic loadgen python _pytorch _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cpu.pickle --modelpath=baseline_8mp_ss_scales_ep15.pth --modelcfg.num_classes=13 --modelcfg.config=baseline_8MP_ss_scales ``` @@ -110,11 +110,13 @@ cmr "get ml-model abtf-ssd-pytorch _skip_weights _skip_code" Other ways to run local (private) model: +You can first copy ABTF model code from GitHub to your local directory `my-model-code`. + ``` -cmr "generic loadgen python _pytorch _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cpu.pickle ^ - --modelpath=baseline_8mp.pth ^ - --modelcfg.num_classes=13 ^ - --modelcodepath="my-model-code" ^ +cmr "generic loadgen python _pytorch _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cpu.pickle \ + --modelpath=baseline_8mp_ss_scales_ep15.pth \ + --modelcfg.num_classes=13 \ + --modelcodepath="my-model-code" \ --modelcfg.config=baseline_8MP_ss_scales ``` From 7829237b45ef2f94704fee8ca7e2be6256b29a29 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 10 Apr 2024 16:50:35 +0200 Subject: [PATCH 09/33] improving readmes --- README-cuda.md | 9 ++++++--- README.md | 12 ++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/README-cuda.md b/README-cuda.md index 3634938..a1fd122 100644 --- a/README-cuda.md +++ b/README-cuda.md @@ -1,14 +1,17 @@ [ [Back to the main page](README.md) ] -*Tested with PyTorch 2.2.2 and CUDA 11.8 and 12.1* ## Prepare workflow to benchmark ABTF model on CUDA-based device -We expect that you already have CUDA driver installed +### Prerequisites + +* We expect that you already have CUDA driver installed +* Tested with PyTorch 2.2.2 and CUDA 11.8 and 12.1 + ### Detect or install CUDA toolkit and libraries -``bash +```bash cmr "get cuda _toolkit _cudnn" cmr "get cuda-devices" ``` diff --git a/README.md b/README.md index 06f58fc..f8bd0b6 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,18 @@ cm pull repo mlcommons@ck --checkout=dev cm pull repo mlcommons@cm4abtf --checkout=dev ``` +### Find repositories + +```bash +cm find repo +``` + +### Update all repositories at any time + +```bash +cm pull repo +``` + ## Download test ABTF model and image From 7587490f120a765cc0da59bd4ab86cfd979fdadb Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 10 Apr 2024 17:20:09 +0200 Subject: [PATCH 10/33] clean up --- README-cuda.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README-cuda.md b/README-cuda.md index a1fd122..dd7560c 100644 --- a/README-cuda.md +++ b/README-cuda.md @@ -9,6 +9,13 @@ * Tested with PyTorch 2.2.2 and CUDA 11.8 and 12.1 +### Build MLPerf loadgen + +```bash +cmr "get mlperf inference loadgen _copy" --version=main +``` + + ### Detect or install CUDA toolkit and libraries ```bash From 2eeb1e40d489a33b9a12eb42de2740978f87f398 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 10 Apr 2024 18:32:57 +0200 Subject: [PATCH 11/33] clean up docs --- README-issues.md | 6 ++++++ README.md | 6 ++++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 README-issues.md diff --git a/README-issues.md b/README-issues.md new file mode 100644 index 0000000..29d5df9 --- /dev/null +++ b/README-issues.md @@ -0,0 +1,6 @@ +# Weird case on Windows + +## 20240410: Grigori + +If I download baseline_8mp_ss_scales_ep15.pth to the ROOT directory with the virtual environment, +pip stops working since it considers this file as a broken package ... diff --git a/README.md b/README.md index 06f58fc..1843234 100644 --- a/README.md +++ b/README.md @@ -21,13 +21,15 @@ and can be removed at any time without influencing your own environment! ```bash python3 -m venv ABTF -. ABTF/bin/activate ; export CM_REPOS=$PWD/ABTF/CM +mkdir ABTF/work +. ABTF/bin/activate ; export CM_REPOS=$PWD/ABTF/CM ; cd ABTF/work ``` ### Windows ```bash python -m venv ABTF -call ABTF\Scripts\activate.bat & set CM_REPOS=%CD%\ABTF\CM +md ABTF\work +call ABTF\Scripts\activate.bat & set CM_REPOS=%CD%\ABTF\CM & cd ABTF\work ``` ## Install CM automations From 0304d25d8b8d7d8973a6e0bed2e00367a82ec09d Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 10 Apr 2024 18:43:56 +0200 Subject: [PATCH 12/33] clean up --- README-cuda.md | 13 +++++++------ README.md | 11 +++++++++-- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/README-cuda.md b/README-cuda.md index dd7560c..974ee89 100644 --- a/README-cuda.md +++ b/README-cuda.md @@ -9,12 +9,6 @@ * Tested with PyTorch 2.2.2 and CUDA 11.8 and 12.1 -### Build MLPerf loadgen - -```bash -cmr "get mlperf inference loadgen _copy" --version=main -``` - ### Detect or install CUDA toolkit and libraries @@ -23,6 +17,13 @@ cmr "get cuda _toolkit _cudnn" cmr "get cuda-devices" ``` +### Build MLPerf loadgen + +```bash +cmr "get mlperf inference loadgen _copy" --version=main +``` + + ### Install or detect PyTorch and PyTorchVision #### CUDA 11.8 diff --git a/README.md b/README.md index b50a1e4..1e3256c 100644 --- a/README.md +++ b/README.md @@ -46,12 +46,19 @@ cm pull repo mlcommons@ck --checkout=dev cm pull repo mlcommons@cm4abtf --checkout=dev ``` -### Find repositories +### Show installed CM repositories ```bash -cm find repo +cm show repo ``` +### Fins a specific repo + +```bash +cm find repo mlcommons@cm4abtf +``` + + ### Update all repositories at any time ```bash From 84421a1eb28486157764094d6a030dc5bf4a49b3 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 10 Apr 2024 18:46:55 +0200 Subject: [PATCH 13/33] clean up --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1e3256c..34b20d2 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ cmr "download file _wget" --url="https://www.dropbox.com/scl/fi/od48qvnbqyfuy1z3 cmr "download file _wget" --url="https://www.dropbox.com/scl/fi/0n7rmxxwqvg04sxk7bbum/0000008766.png?rlkey=mhmr3ztrlsqk8oa67qtxoowuh&dl=0" --verify=no --md5sum=903306a7c8bfbe6c1ca68fad6e34fe52 ``` -### Install ABTF model PyTorch code +### Download ABTF model code and register in CM cache ```bash cmr "get ml-model abtf-ssd-pytorch _local.baseline_8mp_ss_scales_ep15.pth" From d4f0df0f9a1146ea3be6a9aade5c242cb019fb73 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 10 Apr 2024 18:47:37 +0200 Subject: [PATCH 14/33] clean up --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 34b20d2..a91fe0c 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ cm pull repo ``` -## Download test ABTF model and image +### Download ABTF model and sample image ```bash cmr "download file _wget" --url="https://www.dropbox.com/scl/fi/od48qvnbqyfuy1z3aas84/baseline_8mp_ss_scales_ep15.pth?rlkey=d6ybe7g09g21pondmbd3pivzk&dl=0" --verify=no --md5sum=c36cb56b5f6bf8edbe64f9914506e09d From 90efeec6ed2d92790c9bcc9da21346d718ad24ae Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 10 Apr 2024 21:45:46 +0200 Subject: [PATCH 15/33] fixed _repo tag --- script/get-ml-model-abtf-ssd-pytorch/_cm.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml b/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml index 914d338..5b6d986 100644 --- a/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml +++ b/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml @@ -32,7 +32,7 @@ deps: - tags: detect,os -- tags: get,git,repo,_repo.<<>> +- tags: get,git,repo names: - abtf-ssd-pytorch-git-repo - abtf-ml-model-code-git-repo @@ -43,6 +43,9 @@ deps: CM_GIT_CHECKOUT_PATH_ENV_NAME: CM_ABTF_SSD_PYTORCH CM_GIT_BRANCH: <<>> extra_cache_tags: abtf,ssd,pytorch,cm-model + update_tags_from_env_with_prefix: + _repo.: + - CM_ABTF_MODEL_CODE_GIT_URL - tags: download,file,_wget @@ -59,7 +62,7 @@ deps: - 'yes' update_tags_from_env_with_prefix: _url.: - - CM_ML_MODEL_URL + - CM_ML_MODEL_URL From 193d090ae793d49f66fbfae0d75cd57075f393a9 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 10 Apr 2024 21:51:06 +0200 Subject: [PATCH 16/33] fixed abtf model meta --- script/get-ml-model-abtf-ssd-pytorch/_cm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml b/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml index 5b6d986..ecb400c 100644 --- a/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml +++ b/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml @@ -42,7 +42,7 @@ deps: env: CM_GIT_CHECKOUT_PATH_ENV_NAME: CM_ABTF_SSD_PYTORCH CM_GIT_BRANCH: <<>> - extra_cache_tags: abtf,ssd,pytorch,cm-model + extra_cache_tags: abtf,ssd,pytorch,ml-model,cmc update_tags_from_env_with_prefix: _repo.: - CM_ABTF_MODEL_CODE_GIT_URL From e87f4d9ab1f93e8ae2c8e639d135cf913e40e156 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 10 Apr 2024 21:51:20 +0200 Subject: [PATCH 17/33] clean up readme --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a91fe0c..89e9c21 100644 --- a/README.md +++ b/README.md @@ -79,12 +79,18 @@ cmr "download file _wget" --url="https://www.dropbox.com/scl/fi/0n7rmxxwqvg04sxk cmr "get ml-model abtf-ssd-pytorch _local.baseline_8mp_ss_scales_ep15.pth" ``` +### Check CM cache -Check CM cache: ```bash cm show cache ``` +### Find model code in CM cache + +```bash +cm find cache --tags=get,ml-model,abtf,cmc +``` + ## Prepare workflow to benchmark ABTF model on host CPU From d91a51a859a8b9cbc8b6f79ac916c8ec6b0c372c Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 10 Apr 2024 22:16:50 +0200 Subject: [PATCH 18/33] clean up --- script/test-ssd-resnet50-cognata-pytorch/_cm.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml index c16c04b..48689f2 100644 --- a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml +++ b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml @@ -37,6 +37,8 @@ deps: - python3 - tags: get,generic-python-lib,_numpy +- tags: get,generic-python-lib,_package.typing-extensions +- tags: get,generic-python-lib,_package.sympy - tags: get,generic-python-lib,_package.Pillow - tags: get,generic-python-lib,_opencv-python - tags: get,generic-python-lib,_onnx From e6fb6d21bb3dc2d19199def7a31b1c919bff6495 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 12 Apr 2024 13:23:56 +0200 Subject: [PATCH 19/33] working on container demo --- script/test-ssd-resnet50-cognata-pytorch/_cm.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml index c16c04b..dbdc212 100644 --- a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml +++ b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml @@ -117,4 +117,6 @@ docker: skip_input_for_fake_run: - input - output + - config + - num_classes - export_model From e92470aa40f55d28a1eb13facfb25dad527073de Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 12 Apr 2024 13:24:23 +0200 Subject: [PATCH 20/33] working on container demo --- README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a91fe0c..192b472 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Follow [this online guide](https://access.cknowledge.org/playground/?action=inst to install CM for your OS with minimal dependencies. + ## Install virtual environment We suggest to create a virtual environment to avoid messing up your Python installation. @@ -32,6 +33,8 @@ md ABTF\work call ABTF\Scripts\activate.bat & set CM_REPOS=%CD%\ABTF\CM & cd ABTF\work ``` + + ## Install CM automations ### CM for MLOps @@ -79,8 +82,14 @@ cmr "download file _wget" --url="https://www.dropbox.com/scl/fi/0n7rmxxwqvg04sxk cmr "get ml-model abtf-ssd-pytorch _local.baseline_8mp_ss_scales_ep15.pth" ``` +### Check the state of CM cache + +```bash +cm show cache +``` + +### Find model -Check CM cache: ```bash cm show cache ``` From de0a6eb2c6171ef16e46a35c14f6c447e398e61f Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Sat, 13 Apr 2024 09:26:15 +0000 Subject: [PATCH 21/33] moved --gpus=all only for CUDA --- script/test-ssd-resnet50-cognata-pytorch/_cm.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml index e34d51b..1b9023c 100644 --- a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml +++ b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml @@ -93,6 +93,8 @@ variations: CM_DEVICE: cpu cuda: + docker: + all_gpus: 'yes' group: device env: @@ -111,7 +113,6 @@ input_description: docker: skip_run_cmd: 'no' - all_gpus: 'yes' input_paths: - input - output From 7dffa8c1d8053bfb12836e12ef3f4b4a48160983 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Sat, 13 Apr 2024 13:07:19 +0200 Subject: [PATCH 22/33] fixes to suppport containers --- README.md | 4 ++-- script/get-ml-model-abtf-ssd-pytorch/_cm.yaml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 192b472..e0c2fe2 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ cmr "download file _wget" --url="https://www.dropbox.com/scl/fi/0n7rmxxwqvg04sxk cmr "get ml-model abtf-ssd-pytorch _local.baseline_8mp_ss_scales_ep15.pth" ``` -### Check the state of CM cache +### Check the state of the CM cache ```bash cm show cache @@ -91,7 +91,7 @@ cm show cache ### Find model ```bash -cm show cache +cm show cache --tags=ml-model,abtf ``` diff --git a/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml b/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml index ecb400c..045fe81 100644 --- a/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml +++ b/script/get-ml-model-abtf-ssd-pytorch/_cm.yaml @@ -40,6 +40,7 @@ deps: CM_SKIP_MODEL_CODE_DOWNLOAD: - 'yes' env: + CM_GIT_AUTH: 'yes' CM_GIT_CHECKOUT_PATH_ENV_NAME: CM_ABTF_SSD_PYTORCH CM_GIT_BRANCH: <<>> extra_cache_tags: abtf,ssd,pytorch,ml-model,cmc From 73fec8c90a66f12498a74cf6cac9f699e396979b Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Sat, 13 Apr 2024 18:37:31 +0200 Subject: [PATCH 23/33] added deps on CM-MLOps repo --- README-issues.md | 6 - README.md | 158 +---------------- cmr.yaml | 3 + .../test-abtf-model/README-cuda.md | 0 docs/test-abtf-model/README-kb.md | 17 ++ docs/test-abtf-model/README.md | 160 ++++++++++++++++++ docs/test-abtf-model/_cm.json | 11 ++ .../_cm.yaml | 1 + 8 files changed, 195 insertions(+), 161 deletions(-) delete mode 100644 README-issues.md rename README-cuda.md => docs/test-abtf-model/README-cuda.md (100%) create mode 100644 docs/test-abtf-model/README-kb.md create mode 100644 docs/test-abtf-model/README.md create mode 100644 docs/test-abtf-model/_cm.json diff --git a/README-issues.md b/README-issues.md deleted file mode 100644 index 29d5df9..0000000 --- a/README-issues.md +++ /dev/null @@ -1,6 +0,0 @@ -# Weird case on Windows - -## 20240410: Grigori - -If I download baseline_8mp_ss_scales_ep15.pth to the ROOT directory with the virtual environment, -pip stops working since it considers this file as a broken package ... diff --git a/README.md b/README.md index e0c2fe2..ce2d89d 100644 --- a/README.md +++ b/README.md @@ -1,160 +1,8 @@ # Collective Mind interface and automation for ABTF -This repository contains [MLCommons CM](https://github.com/mlcommons/ck) automation recipes +This repository contains [CM scripts (cross-platform automation recipes)](https://github.com/mlcommons/ck) to make it easier to prepare and benchmark different versions of ABTF models (public or private) with MLPerf loadgen across different software and hardware. - -## Install MLCommons CM automation framework - -Follow [this online guide](https://access.cknowledge.org/playground/?action=install) -to install CM for your OS with minimal dependencies. - - - -## Install virtual environment - -We suggest to create a virtual environment to avoid messing up your Python installation. -All CM repositories, artifacts and cache will be resided inside this virtual environment -and can be removed at any time without influencing your own environment! - -### Linux - -```bash -python3 -m venv ABTF -mkdir ABTF/work -. ABTF/bin/activate ; export CM_REPOS=$PWD/ABTF/CM ; cd ABTF/work -``` -### Windows - -```bash -python -m venv ABTF -md ABTF\work -call ABTF\Scripts\activate.bat & set CM_REPOS=%CD%\ABTF\CM & cd ABTF\work -``` - - - -## Install CM automations - -### CM for MLOps - -```bash -cm pull repo mlcommons@ck --checkout=dev -``` - -### CM for ABTF - -```bash -cm pull repo mlcommons@cm4abtf --checkout=dev -``` - -### Show installed CM repositories - -```bash -cm show repo -``` - -### Fins a specific repo - -```bash -cm find repo mlcommons@cm4abtf -``` - - -### Update all repositories at any time - -```bash -cm pull repo -``` - - -### Download ABTF model and sample image - -```bash -cmr "download file _wget" --url="https://www.dropbox.com/scl/fi/od48qvnbqyfuy1z3aas84/baseline_8mp_ss_scales_ep15.pth?rlkey=d6ybe7g09g21pondmbd3pivzk&dl=0" --verify=no --md5sum=c36cb56b5f6bf8edbe64f9914506e09d -cmr "download file _wget" --url="https://www.dropbox.com/scl/fi/0n7rmxxwqvg04sxk7bbum/0000008766.png?rlkey=mhmr3ztrlsqk8oa67qtxoowuh&dl=0" --verify=no --md5sum=903306a7c8bfbe6c1ca68fad6e34fe52 -``` - -### Download ABTF model code and register in CM cache - -```bash -cmr "get ml-model abtf-ssd-pytorch _local.baseline_8mp_ss_scales_ep15.pth" -``` - -### Check the state of the CM cache - -```bash -cm show cache -``` - -### Find model - -```bash -cm show cache --tags=ml-model,abtf -``` - - -## Prepare workflow to benchmark ABTF model on host CPU - -Next commands prepare environment to benchmark host CPU. -Check these docs to benchmark other devices: -* [CUDA-based device](README-cuda.md) - -### Build MLPerf loadgen - -```bash -cmr "get mlperf inference loadgen _copy" --version=main -``` - -### Install or detect PyTorch - -```bash -cmr "get generic-python-lib _torch" -cmr "get generic-python-lib _torchvision" -``` - - - -## Test Model with a test image - -```bash -cmr "test abtf ssd-resnet50 cognata pytorch" --input=0000008766.png --output=0000008766_prediction_test.jpg --config=baseline_8MP_ss_scales --num-classes=13 -``` - -## Benchmark model with MLPerf loadgen - -```bash -cmr "generic loadgen python _pytorch _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cpu.pickle --modelpath=baseline_8mp_ss_scales_ep15.pth --modelcfg.num_classes=13 --modelcfg.config=baseline_8MP_ss_scales -``` - - -## Benchmarking other models - -Other ways to download public or private model code and weights: -```bash -cmr "get ml-model abtf-ssd-pytorch _skip_weights" --adr.abtf-ml-model-code-git-repo.env.CM_ABTF_MODEL_CODE_GIT_URL=https://github.com/mlcommons/abtf-ssd-pytorch -cmr "get ml-model abtf-ssd-pytorch _skip_weights" --model_code_git_url=https://github.com/mlcommons/abtf-ssd-pytorch --model_code_git_branch=cognata-cm -cmr "get ml-model abtf-ssd-pytorch _skip_weights _skip_code" -``` - -Other ways to run local (private) model: - -You can first copy ABTF model code from GitHub to your local directory `my-model-code`. - -``` -cmr "generic loadgen python _pytorch _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cpu.pickle \ - --modelpath=baseline_8mp_ss_scales_ep15.pth \ - --modelcfg.num_classes=13 \ - --modelcodepath="my-model-code" \ - --modelcfg.config=baseline_8MP_ss_scales -``` - - - - - -## Feedback - -Join MLCommons discord or get in touch with developer: gfursin@cknowledge.org - +* Run and benchmark reference ABTF model via CM (CPU and CUDA): [README](docs/test-abtf-model/README.md) +* Knowledge base: [README](docs/test-abtf-model/README-kb.md) \ No newline at end of file diff --git a/cmr.yaml b/cmr.yaml index 3088cbe..7fa9333 100644 --- a/cmr.yaml +++ b/cmr.yaml @@ -2,3 +2,6 @@ alias: mlcommons@cm4abtf uid: 566d31eda11948a9 git: true + +deps: + - name: mlcommons@ck diff --git a/README-cuda.md b/docs/test-abtf-model/README-cuda.md similarity index 100% rename from README-cuda.md rename to docs/test-abtf-model/README-cuda.md diff --git a/docs/test-abtf-model/README-kb.md b/docs/test-abtf-model/README-kb.md new file mode 100644 index 0000000..6a471ff --- /dev/null +++ b/docs/test-abtf-model/README-kb.md @@ -0,0 +1,17 @@ +# Issues + +## Weird case on Windows + +### 20240410: Grigori + +If I download baseline_8mp_ss_scales_ep15.pth to the ROOT directory with the virtual environment, +pip stops working since it considers this file as a broken package ... + + +# Misc commands + +Register local ABTF model in CM cache to be the default + +```bash +cmr "get ml-model abtf-ssd-pytorch _local.baseline_8mp_ss_scales_ep15.pth" +``` diff --git a/docs/test-abtf-model/README.md b/docs/test-abtf-model/README.md new file mode 100644 index 0000000..763f461 --- /dev/null +++ b/docs/test-abtf-model/README.md @@ -0,0 +1,160 @@ +# Collective Mind interface and automation for ABTF + +This repository contains [MLCommons CM](https://github.com/mlcommons/ck) automation recipes +to make it easier to prepare and benchmark different versions of ABTF models +(public or private) with MLPerf loadgen across different software and hardware. + + +## Install MLCommons CM automation framework + +Follow [this online guide](https://access.cknowledge.org/playground/?action=install) +to install CM for your OS with minimal dependencies. + + + +## Install virtual environment + +We suggest to create a virtual environment to avoid messing up your Python installation. +All CM repositories, artifacts and cache will be resided inside this virtual environment +and can be removed at any time without influencing your own environment! + +### Linux + +```bash +python3 -m venv ABTF +mkdir ABTF/work +. ABTF/bin/activate ; export CM_REPOS=$PWD/ABTF/CM ; cd ABTF/work +``` +### Windows + +```bash +python -m venv ABTF +md ABTF\work +call ABTF\Scripts\activate.bat & set CM_REPOS=%CD%\ABTF\CM & cd ABTF\work +``` + + + +## Install CM automations + +### CM for MLOps + +```bash +cm pull repo mlcommons@ck --checkout=dev +``` + +### CM for ABTF + +```bash +cm pull repo mlcommons@cm4abtf --checkout=dev +``` + +### Show installed CM repositories + +```bash +cm show repo +``` + +### Fins a specific repo + +```bash +cm find repo mlcommons@cm4abtf +``` + + +### Update all repositories at any time + +```bash +cm pull repo +``` + + +### Download ABTF model and sample image + +```bash +cmr "download file _wget" --url="https://www.dropbox.com/scl/fi/od48qvnbqyfuy1z3aas84/baseline_8mp_ss_scales_ep15.pth?rlkey=d6ybe7g09g21pondmbd3pivzk&dl=0" --verify=no --md5sum=c36cb56b5f6bf8edbe64f9914506e09d +cmr "download file _wget" --url="https://www.dropbox.com/scl/fi/0n7rmxxwqvg04sxk7bbum/0000008766.png?rlkey=mhmr3ztrlsqk8oa67qtxoowuh&dl=0" --verify=no --md5sum=903306a7c8bfbe6c1ca68fad6e34fe52 +``` + +### Download ABTF model code and register in CM cache + +```bash +cmr "get ml-model abtf-ssd-pytorch _local.baseline_8mp_ss_scales_ep15.pth" +``` + +### Check the state of the CM cache + +```bash +cm show cache +``` + +### Find model code + +```bash +cm show cache --tags=ml-model,abtf +``` + + +## Prepare workflow to benchmark ABTF model on host CPU + +Next commands prepare environment to benchmark host CPU. +Check these docs to benchmark other devices: +* [CUDA-based device](README-cuda.md) + +### Build MLPerf loadgen + +```bash +cmr "get mlperf inference loadgen _copy" --version=main +``` + +### Install or detect PyTorch + +```bash +cmr "get generic-python-lib _torch" +cmr "get generic-python-lib _torchvision" +``` + + + +## Test Model with a test image + +```bash +cmr "test abtf ssd-resnet50 cognata pytorch" --input=0000008766.png --output=0000008766_prediction_test.jpg --config=baseline_8MP_ss_scales --num-classes=13 +``` + +## Benchmark model with MLPerf loadgen + +```bash +cmr "generic loadgen python _pytorch _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cpu.pickle --modelpath=baseline_8mp_ss_scales_ep15.pth --modelcfg.num_classes=13 --modelcfg.config=baseline_8MP_ss_scales +``` + + +## Benchmarking other models + +Other ways to download public or private model code and weights: +```bash +cmr "get ml-model abtf-ssd-pytorch _skip_weights" --adr.abtf-ml-model-code-git-repo.env.CM_ABTF_MODEL_CODE_GIT_URL=https://github.com/mlcommons/abtf-ssd-pytorch +cmr "get ml-model abtf-ssd-pytorch _skip_weights" --model_code_git_url=https://github.com/mlcommons/abtf-ssd-pytorch --model_code_git_branch=cognata-cm +cmr "get ml-model abtf-ssd-pytorch _skip_weights _skip_code" +``` + +Other ways to run local (private) model: + +You can first copy ABTF model code from GitHub to your local directory `my-model-code`. + +``` +cmr "generic loadgen python _pytorch _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cpu.pickle \ + --modelpath=baseline_8mp_ss_scales_ep15.pth \ + --modelcfg.num_classes=13 \ + --modelcodepath="my-model-code" \ + --modelcfg.config=baseline_8MP_ss_scales +``` + + + + + +## Feedback + +Join MLCommons discord or get in touch with developer: gfursin@cknowledge.org + diff --git a/docs/test-abtf-model/_cm.json b/docs/test-abtf-model/_cm.json new file mode 100644 index 0000000..6d0ded0 --- /dev/null +++ b/docs/test-abtf-model/_cm.json @@ -0,0 +1,11 @@ +{ + "alias": "test-abtf-model", + "automation_alias": "docs", + "automation_uid": "9558c9e6ca124065", + "tags": [ + "test", + "abtf", + "model" + ], + "uid": "3796184894ac4dd0" +} diff --git a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml index 1b9023c..51e2129 100644 --- a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml +++ b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml @@ -19,6 +19,7 @@ input_mapping: input: CM_INPUT_IMAGE output: CM_OUTPUT_IMAGE export_model: CM_ABTF_EXPORT_MODEL_TO_ONNX + model: CM_ML_MODEL_FILE_WITH_PATH config: CM_ABTF_ML_MODEL_CONFIG dataset: CM_ABTF_DATASET num_classes: CM_ABTF_NUM_CLASSES From 09e8c63105c4972a939d50c3d38b1042f3717409 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Sat, 13 Apr 2024 18:44:26 +0200 Subject: [PATCH 24/33] improving support for Docker --- script/test-ssd-resnet50-cognata-pytorch/_cm.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml index 51e2129..db1f1c3 100644 --- a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml +++ b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml @@ -118,9 +118,11 @@ docker: - input - output - export_model + - model skip_input_for_fake_run: - input - output + - model - config - num_classes - export_model From 627de0386911fe3aecfe63f61a999180e5f1b023 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Mon, 15 Apr 2024 09:25:49 +0000 Subject: [PATCH 25/33] added requirement for GH private token and extra repo when using docker --- script/test-ssd-resnet50-cognata-pytorch/_cm.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml index db1f1c3..77e2da4 100644 --- a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml +++ b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml @@ -114,11 +114,13 @@ input_description: docker: skip_run_cmd: 'no' + gh_token_required: True + cm_repos: 'cm pull repo --url=https://oauth2:${CM_GH_TOKEN}@github.com/mlcommons/cm4abtf.git --checkout=dev' input_paths: - input + - model - output - export_model - - model skip_input_for_fake_run: - input - output From 69574606f1039e6fd1506dbce524ae555d2ea521 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Mon, 15 Apr 2024 18:04:24 +0200 Subject: [PATCH 26/33] clean up --- script/test-ssd-resnet50-cognata-pytorch/_cm.yaml | 3 +-- script/test-ssd-resnet50-cognata-pytorch/customize.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml index 77e2da4..77f12e7 100644 --- a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml +++ b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml @@ -96,8 +96,7 @@ variations: cuda: docker: all_gpus: 'yes' - group: - device + group: device env: CM_DEVICE: cuda diff --git a/script/test-ssd-resnet50-cognata-pytorch/customize.py b/script/test-ssd-resnet50-cognata-pytorch/customize.py index e752428..88306c6 100644 --- a/script/test-ssd-resnet50-cognata-pytorch/customize.py +++ b/script/test-ssd-resnet50-cognata-pytorch/customize.py @@ -18,7 +18,7 @@ def preprocess(i): if extra!='': print ('') print ('Extra command line: {}'.format(extra)) - + env['CM_ABTF_EXTRA_CMD'] = extra return {'return':0} From ccf03d0dd2de16ec773e3a0865f0ca2477e6f3c5 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Mon, 15 Apr 2024 21:10:26 +0200 Subject: [PATCH 27/33] added default Docker base --- script/test-ssd-resnet50-cognata-pytorch/_cm.yaml | 1 + script/test-ssd-resnet50-cognata-pytorch/customize.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml index 77e2da4..5b02fc5 100644 --- a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml +++ b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml @@ -96,6 +96,7 @@ variations: cuda: docker: all_gpus: 'yes' + base_image: nvcr.io/nvidia/pytorch:24.03-py3 group: device env: diff --git a/script/test-ssd-resnet50-cognata-pytorch/customize.py b/script/test-ssd-resnet50-cognata-pytorch/customize.py index e752428..88306c6 100644 --- a/script/test-ssd-resnet50-cognata-pytorch/customize.py +++ b/script/test-ssd-resnet50-cognata-pytorch/customize.py @@ -18,7 +18,7 @@ def preprocess(i): if extra!='': print ('') print ('Extra command line: {}'.format(extra)) - + env['CM_ABTF_EXTRA_CMD'] = extra return {'return':0} From d8f8d93fbd6c954492c0abe0f16707443cb03d3c Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 17 Apr 2024 15:39:53 +0200 Subject: [PATCH 28/33] added CM repo deps and antideps --- cmr.yaml | 3 +++ docs/test-abtf-model/README.md | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cmr.yaml b/cmr.yaml index 7fa9333..607c4be 100644 --- a/cmr.yaml +++ b/cmr.yaml @@ -4,4 +4,7 @@ uid: 566d31eda11948a9 git: true deps: + - name: mlcommons@cm4mlops - name: mlcommons@ck + conflict: True + diff --git a/docs/test-abtf-model/README.md b/docs/test-abtf-model/README.md index 763f461..2d603fc 100644 --- a/docs/test-abtf-model/README.md +++ b/docs/test-abtf-model/README.md @@ -40,7 +40,7 @@ call ABTF\Scripts\activate.bat & set CM_REPOS=%CD%\ABTF\CM & cd ABTF\work ### CM for MLOps ```bash -cm pull repo mlcommons@ck --checkout=dev +cm pull repo mlcommons@cm4mlops --checkout=dev ``` ### CM for ABTF From c179e428ef65f1c44e61a1a746a461e066e9b11c Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 17 Apr 2024 16:04:41 +0200 Subject: [PATCH 29/33] updated deps --- cmr.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmr.yaml b/cmr.yaml index 607c4be..f290512 100644 --- a/cmr.yaml +++ b/cmr.yaml @@ -4,7 +4,10 @@ uid: 566d31eda11948a9 git: true deps: - - name: mlcommons@cm4mlops - - name: mlcommons@ck + - alias: mlcommons@cm4mlops + uid: 9e97bb72b0474657 + + - alias: mlcommons@ck + uid: a4705959af8e447a conflict: True From ab848c8d7d5b9e74d7b3fbb1d56b7c95d3e501a7 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 19 Apr 2024 12:29:52 +0200 Subject: [PATCH 30/33] working on Cognata dataset --- .../get-dataset-cognata-mlcommons/README.md | 187 ------------------ script/get-dataset-cognata-mlcommons/_cm.yaml | 35 +--- .../customize.py | 39 +--- .../get-dataset-cognata-mlcommons/license.txt | 107 ---------- 4 files changed, 11 insertions(+), 357 deletions(-) delete mode 100644 script/get-dataset-cognata-mlcommons/README.md delete mode 100644 script/get-dataset-cognata-mlcommons/license.txt diff --git a/script/get-dataset-cognata-mlcommons/README.md b/script/get-dataset-cognata-mlcommons/README.md deleted file mode 100644 index 606d817..0000000 --- a/script/get-dataset-cognata-mlcommons/README.md +++ /dev/null @@ -1,187 +0,0 @@ -
-Click here to see the table of contents. - -* [About](#about) -* [Summary](#summary) -* [Reuse this script in your project](#reuse-this-script-in-your-project) - * [ Install CM automation language](#install-cm-automation-language) - * [ Check CM script flags](#check-cm-script-flags) - * [ Run this script from command line](#run-this-script-from-command-line) - * [ Run this script from Python](#run-this-script-from-python) - * [ Run this script via GUI](#run-this-script-via-gui) - * [ Run this script via Docker (beta)](#run-this-script-via-docker-(beta)) -* [Customization](#customization) - * [ Variations](#variations) - * [ Default environment](#default-environment) -* [Script workflow, dependencies and native scripts](#script-workflow-dependencies-and-native-scripts) -* [Script output](#script-output) -* [New environment keys (filter)](#new-environment-keys-(filter)) -* [New environment keys auto-detected from customize](#new-environment-keys-auto-detected-from-customize) -* [Maintainers](#maintainers) - -
- -*Note that this README is automatically generated - don't edit!* - -### About - -#### Summary - -* Category: *AI/ML datasets.* -* CM GitHub repository: *[mlcommons@ck](https://github.com/mlcommons/ck/tree/master/cm-mlops)* -* GitHub directory for this script: *[GitHub](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-dataset-cognata)* -* CM meta description for this script: *[_cm.json](_cm.json)* -* CM "database" tags to find this script: *get,dataset,cognata,object-detection,original* -* Output cached? *True* -___ -### Reuse this script in your project - -#### Install CM automation language - -* [Installation guide](https://github.com/mlcommons/ck/blob/master/docs/installation.md) -* [CM intro](https://doi.org/10.5281/zenodo.8105339) - -#### Pull CM repository with this automation - -```cm pull repo mlcommons@ck``` - - -#### Run this script from command line - -1. `cm run script --tags=get,dataset,cognata,object-detection,original[,variations] ` - -2. `cmr "get dataset cognata object-detection original[ variations]" ` - -* `variations` can be seen [here](#variations) - -#### Run this script from Python - -
-Click here to expand this section. - -```python - -import cmind - -r = cmind.access({'action':'run' - 'automation':'script', - 'tags':'get,dataset,cognata,object-detection,original' - 'out':'con', - ... - (other input keys for this script) - ... - }) - -if r['return']>0: - print (r['error']) - -``` - -
- - -#### Run this script via GUI - -```cmr "cm gui" --script="get,dataset,cognata,object-detection,original"``` - -Use this [online GUI](https://cKnowledge.org/cm-gui/?tags=get,dataset,cognata,object-detection,original) to generate CM CMD. - -#### Run this script via Docker (beta) - -`cm docker script "get dataset cognata object-detection original[ variations]" ` - -___ -### Customization - - -#### Variations - - * Group "**dataset-type**" -
- Click here to expand this section. - - * `_calibration` - - Workflow: - * **`_validation`** (default) - - Environment variables: - - *CM_DATASET_CALIBRATION*: `no` - - Workflow: - -
- - - * Group "**size**" -
- Click here to expand this section. - - * **`_50`** (default) - - Environment variables: - - *CM_DATASET_SIZE*: `50` - - Workflow: - * `_500` - - Environment variables: - - *CM_DATASET_SIZE*: `500` - - Workflow: - * `_full` - - Environment variables: - - *CM_DATASET_SIZE*: `` - - Workflow: - * `_size.#` - - Environment variables: - - *CM_DATASET_SIZE*: `#` - - Workflow: - -
- - -#### Default variations - -`_50,_validation` -#### Default environment - -
-Click here to expand this section. - -These keys can be updated via `--env.KEY=VALUE` or `env` dictionary in `@input.json` or using script flags. - -* CM_DATASET_CALIBRATION: `no` - -
- -___ -### Script workflow, dependencies and native scripts - -
-Click here to expand this section. - - 1. ***Read "deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-dataset-cognata/_cm.json)*** - * get,python3 - * CM names: `--adr.['python', 'python3']...` - - CM script: [get-python3](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-python3) - 1. ***Run "preprocess" function from [customize.py](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-dataset-cognata/customize.py)*** - 1. Read "prehook_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-dataset-cognata/_cm.json) - 1. ***Run native script if exists*** - 1. Read "posthook_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-dataset-cognata/_cm.json) - 1. ***Run "postrocess" function from [customize.py](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-dataset-cognata/customize.py)*** - 1. Read "post_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-dataset-cognata/_cm.json) -
- -___ -### Script output -`cmr "get dataset cognata object-detection original[,variations]" -j` -#### New environment keys (filter) - -* `CM_CALIBRATION_DATASET_PATH` -* `CM_DATASET_ANNOTATIONS_DIR_PATH` -* `CM_DATASET_ANNOTATIONS_FILE_PATH` -* `CM_DATASET_PATH` -* `CM_DATASET_PATH_ROOT` -#### New environment keys auto-detected from customize - -* `CM_CALIBRATION_DATASET_PATH` -* `CM_DATASET_PATH` -* `CM_DATASET_PATH_ROOT` -___ -### Maintainers - -* [Open MLCommons taskforce on automation and reproducibility](https://github.com/mlcommons/ck/blob/master/docs/taskforce.md) \ No newline at end of file diff --git a/script/get-dataset-cognata-mlcommons/_cm.yaml b/script/get-dataset-cognata-mlcommons/_cm.yaml index b2bf5c4..9b1a0fd 100644 --- a/script/get-dataset-cognata-mlcommons/_cm.yaml +++ b/script/get-dataset-cognata-mlcommons/_cm.yaml @@ -6,14 +6,11 @@ automation_uid: 5b4e0237da074764 private: true -cache: true +cache: false category: AI/ML datasets category_sort: 8500 -default_env: - CM_DATASET_CALIBRATION: 'no' - deps: - names: - python @@ -24,6 +21,7 @@ env: CM_DATASET: COGNATA new_env_keys: +- CM_MLCOMMONS_COGNATA_* - CM_DATASET_PATH - CM_DATASET_PATH_ROOT - CM_DATASET_ANNOTATIONS_DIR_PATH @@ -32,34 +30,7 @@ new_env_keys: tags: - get +- raw - dataset - mlcommons-cognata - object-detection -- original - -variations: - '50': - default: true - env: - CM_DATASET_SIZE: '50' - group: size - '500': - env: - CM_DATASET_SIZE: '500' - group: size - calibration: - env: {} - group: dataset-type - full: - env: - CM_DATASET_SIZE: '' - group: size - size.#: - env: - CM_DATASET_SIZE: '#' - group: size - validation: - default: true - env: - CM_DATASET_CALIBRATION: 'no' - group: dataset-type diff --git a/script/get-dataset-cognata-mlcommons/customize.py b/script/get-dataset-cognata-mlcommons/customize.py index 6c55731..122f76f 100644 --- a/script/get-dataset-cognata-mlcommons/customize.py +++ b/script/get-dataset-cognata-mlcommons/customize.py @@ -1,29 +1,6 @@ from cmind import utils import os -import shutil -import sys - - -def ask_user(question, default="yes"): - valid = ["yes", "y", "no", "n"] - if default is None: - prompt = " [y/n] " - elif default == "yes": - prompt = " [Y/n] " - elif default == "no": - prompt = " [y/N] " - else: - raise ValueError("invalid default answer: '%s'" % default) - - while True: - sys.stdout.write(question + prompt) - choice = input().lower() - if default is not None and choice == "": - return default.startswith('y') - elif choice in valid: - return choice.startswith('y') - else: - print("Please input y/n\n") + def preprocess(i): @@ -31,19 +8,19 @@ def preprocess(i): script_path = i['run_script_input']['path'] - with open(os.path.join(script_path, "license.txt"), "r") as f: - print(f.read()) - - response = ask_user("Do you accept?") - - print(response) + input('xyz11') return {'return': 0} def postprocess(i): env = i['env'] - return {'return': -1} #todo + + input('xyz22') + + return {'return': 1, 'error':'TBD'} #todo + + if env.get('CM_DATASET_CALIBRATION','') == "no": env['CM_DATASET_PATH_ROOT'] = os.path.join(os.getcwd(), 'install') env['CM_DATASET_PATH'] = os.path.join(os.getcwd(), 'install', 'validation', 'data') diff --git a/script/get-dataset-cognata-mlcommons/license.txt b/script/get-dataset-cognata-mlcommons/license.txt deleted file mode 100644 index f46fa42..0000000 --- a/script/get-dataset-cognata-mlcommons/license.txt +++ /dev/null @@ -1,107 +0,0 @@ -DATA END USER LICENSE AGREEMENT - -This Data End User License Agreement (“Agreement”) is by and between MLCommons, a Delaware non- -profit corporation (“MLC”) and the undersigned MLCommons Member that is downloading or using the - -MLCommons Cognata Dataset made available by MLCommons (“Licensee”). This Agreement will be -effective upon the date last signed, as reflected in the signature block below. -1. Background. MLC, an open engineering consortium, has licensed a dataset known as MLCommons -Cognata Dataset (the “Data”) which it wishes to make available to its Members in accordance with -the terms set forth in this Agreement. The term Data refers collectively to the MLCommons Cognata -datasets and materials delivered by MLC as a whole, to subsets of such datasets and materials, as -well as to the separate images and annotations comprising such datasets and materials. MLC agrees -that, to the best of its knowledge, the Data will not include any data that is “personal data” or -“personally identifiable” according to any applicable law or regulation. -2. Data License. Subject to the License Restrictions set forth in Section 4 below, MLC grants to Licensee -a non-exclusive, non-transferable, non-sublicenseable worldwide license during the term of the -Agreement to download, install, copy, modify, distribute, publicly display, prepare derivative works, -and otherwise use the Data for the purpose of benchmarking. The rights and license granted under -this Section 2.1 shall be referred to hereinafter as the “License.” -3. Derivative Work, Applications and Output. The License includes the right to create derivative -works of the Data. Any use of the Data incorporated in these derivative works will be governed by -the provisions of this Agreement, however, MLC and its licensors will not be responsible for any -adaptations, revisions, transformations, alterations, or any other modifications of the Data. The -License includes the right to incorporate and use the Data in applications (the “Applications”), -provided that all Data incorporated or used by these Applications is and will remain governed by the -provisions of this Agreement. The term “Output” refers to any analysis, results and calculations -arrived at from the use of the Data, and to any changes to the models and networks that will be -trained using the Data. -4. License Restrictions. -a. The License will only remain in effect so long as Licensee remains an MLC member in good -standing. -b. Licensee may not provide the Data as a whole, or on a standalone basis, to third parties but may -distribute the Applications and Output in the course of Licensee’s ordinary course business. This -includes the right to distribute a subset of the Data in Licensee’s Applications to run relevant -benchmarks generated from the Output. -c. While the License includes the right to use the Data in commercial activity such as -benchmarking, marketing, customer demonstrations, bidding and requests for proposals or -information, it excludes the right to use the Data directly in connection with models used in or -items in production, including training production models. -d. Licensee acknowledges that MLC’s data licensor, Cognata Ltd, is the owner of all right, title and -interest in the Data. It will acknowledge and identify “Cognata” as a data provider in connection -with any distribution, display or publication of the Data, Applications, or Output. -e. Licensee will ensure that its use of the Data will be done in accordance with all applicable law. - - -5. Term. The License will continue in effect until terminated as set forth below: -a. Licensee may terminate the License by informing MLC that all Data, Output and Applications -that incorporate the Data in Licensee’s possession or control have been deleted. -b. MLC may terminate the License upon providing Licensee with at least thirty (30) days prior -notice of termination sent to the email address of the primary contact associated with -Licensee’s Membership in MLC. -c. The License will terminate immediately and automatically without any notice if Licensee ceases -to be an MLC Member or if Licensee violates any of the terms and conditions of this Agreement. -Upon termination of this Agreement, Licensee must delete all copies of the Data and Output in its -possession and cease distributing any Data, Output or Applications that incorporate the Data. -Termination will not limit any of MLC’s rights or remedies at law or in equity. Sections 8 through 11 -of this Agreement, inclusive, shall survive such termination. -6. Storage. For the duration of this Agreement, Licensee may download and store local copies of the -Data on systems under its control. -7. License. This Agreement provides certain license rights to use of the Data but does not constitute a -sale of the Data. MLC’s licensor will continue to own all right, title and interest in the Data. -8. Indemnification. Licensee agrees to indemnify and hold MLC and its licensors, and their respective -officers, directors and agents harmless from any claims, expenses, or liabilities caused by Licensee’s -use of the Data or Licensee’s publication or use of Output; except that Licensee shall have no liability -or obligation to indemnify MLC or its licensor from or against any claims, expenses, or liabilities -resulting from (a) third party claims that the Data infringes, misappropriation, or otherwise violates -third party intellectual property or other rights or (b) MLC’s negligent acts or omissions or willful -misconduct. Licensee will promptly inform MLC in writing of any claim by a third party that use of -the Data in accordance with this Agreement violates the proprietary rights of such claimant -(including any threat, warning or notice prior to such claim in which the claimant alleges that the -Data infringes his/her/its intellectual property rights), which notice will include copies of all available -material related to the claim and will indicate the estimated amount claimed by the claimant, if any. -9. Disclaimers & Limited Liability. -EXCEPT AS EXPLICITLY SET FORTH IN THIS AGREEMENT, THE DATA IS PROVIDED “AS IS,” WITHOUT -WARRANTY OR REPRESENTATION OF ANY KIND WHETHER EXPRESS, IMPLIED, STATUTORY, OR -OTHER. THIS INCLUDES, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, -ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT KNOWN OR -DISCOVERABLE. LICENSEES ARE RESPONSIBLE FOR ANY AND ALL CONCLUSIONS, DECISIONS AND/OR -ACTIONS TAKEN BY THEM IN RELIANCE ON THE DATA. LICENSEE ASSUMES ALL RESPONSIBILITIES -FOR ANY USE OF THE OUTPUT. IN NO EVENT WILL COGNATA LTD, MLC’S DATA LICENSOR, -RESPONSIBLE TO LICENSEE IN ANY WAY FOR THE DATA. -EXCEPT FOR MATTERS FOR WHICH LIABILITY CANNOT BE EXCLUDED OR LIMITED UNDER APPLICABLE -LAW: (A) IN NO EVENT SHALL EITHER PARTY BE LIABLE FOR INCIDENTAL, INDIRECT, SPECIAL, -EXEMPLARY, PUNITIVE AND/OR CONSEQUENTIAL DAMAGES (HOWEVER ARISING) AND/OR FOR -LOSS OF PROFIT, LOSS OF USE, LOSS OF DATA, LOSS OF REVENUES, LOSS OF SAVINGS, BUSINESS -INTERRUPTION, OR LOSS OF REPUTATION ARISING FROM AND/OR IN CONNECTION WITH THIS -AGREEMENT, AND/OR THE USE OF THE DATA AND (B) THE MAXIMUM AGGREGATE LIABILITY OF - - -EITHER PARTY FOR ANY AND ALL DAMAGES AND LOSSES ARISING FROM AND/OR IN CONNECTION -WITH THIS AGREEMENT, AND/OR THE USE OF THE DATA SHALL NOT EXCEED THE AMOUNTS PAID -FOR THE APPLICABLE DATA. -LICENSEE AGREES THAT IT IS NOT ENTITLED TO RECOVER ANY OTHER DAMAGES OR LOSSES EVEN IF -THE ABOVE DAMAGES REMEDY DOESN'T FULLY COMPENSATE LICENSEE FOR ANY DAMAGES OR -LOSSES OR FAILS OF ITS ESSENTIAL PURPOSE AND EVEN IF MLC KNEW OR SHOULD HAVE KNOWN -ABOUT THE POSSIBILITY OF THE DAMAGES OR LOSSES. THE ABOVE LIMITATIONS AND EXCLUSIONS -OF LIABILITY SHALL APPLY TO ALL CAUSES OF ACTION AND REGARDLESS OF THE THEORY OF -LIABILITY (CONTRACT, TORT, EQUITY, BREACH OF STATUTORY DUTY, STRICT LIABILITY OR -OTHERWISE) AND EVEN IF SUCH DAMAGES OR LOSSES WERE OR COULD HAVE BEEN FORESEEABLE. -10. Amendment. MLC may update this Agreement upon 30 days’ written notice to Licensee. -11. Dispute resolution. This Agreement will be governed by the laws of the State of Delaware, without -reference to conflict of laws principles. If unable to resolve a dispute amicably, the parties agree that -disputes arising out of or related to this Agreement will be subject to the exclusive jurisdiction of the -state and federal courts located in San Francisco, California. -IN WITNESS WHEREOF, the undersigned parties have executed this Data End User License Agreement as -of the dates set forth below their signatures. From 1140a911882459f3ee3bc892fce3e2fb447b3a72 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 19 Apr 2024 16:17:24 +0200 Subject: [PATCH 31/33] created cognata automation --- .../README-extra.md | 2 + script/get-dataset-cognata-mlcommons/_cm.yaml | 25 +- .../customize.py | 289 +++++++++++++++++- .../_cm.yaml | 5 +- 4 files changed, 299 insertions(+), 22 deletions(-) create mode 100644 script/get-dataset-cognata-mlcommons/README-extra.md diff --git a/script/get-dataset-cognata-mlcommons/README-extra.md b/script/get-dataset-cognata-mlcommons/README-extra.md new file mode 100644 index 0000000..3edbe8c --- /dev/null +++ b/script/get-dataset-cognata-mlcommons/README-extra.md @@ -0,0 +1,2 @@ +cmr "get raw dataset mlcommons-cognata" --serial_numbers=10002_Urban_Clear_Morning +cmr "get raw dataset mlcommons-cognata" --serial_numbers=10002_Urban_Clear_Morning --group_names=Cognata_Camera_02_8M diff --git a/script/get-dataset-cognata-mlcommons/_cm.yaml b/script/get-dataset-cognata-mlcommons/_cm.yaml index 9b1a0fd..dbaf550 100644 --- a/script/get-dataset-cognata-mlcommons/_cm.yaml +++ b/script/get-dataset-cognata-mlcommons/_cm.yaml @@ -17,20 +17,29 @@ deps: - python3 tags: get,python3 +- tags: get,generic-python-lib,_package.openpyxl + +- tags: get,aria2 + +- names: + - custom-cache-entry + tags: create,custom,cache,entry + extra_cache_tags: dataset,mlcommons-cognata + env_key: MLCOMMONS_COGNATA_DATASET + +input_mapping: + serial_numbers: CM_MLCOMMONS_COGNATA_SERIAL_NUMBERS + group_names: CM_MLCOMMONS_COGNATA_GROUP_NAMES + env: - CM_DATASET: COGNATA + CM_DATASET: MLCOMMONS_COGNATA + CM_MLCOMMONS_COGNATA_KEY1: "Dataset 1.0" new_env_keys: -- CM_MLCOMMONS_COGNATA_* -- CM_DATASET_PATH -- CM_DATASET_PATH_ROOT -- CM_DATASET_ANNOTATIONS_DIR_PATH -- CM_DATASET_ANNOTATIONS_FILE_PATH -- CM_CALIBRATION_DATASET_PATH +- CM_DATASET_MLCOMMONS_COGNATA* tags: - get - raw - dataset - mlcommons-cognata -- object-detection diff --git a/script/get-dataset-cognata-mlcommons/customize.py b/script/get-dataset-cognata-mlcommons/customize.py index 122f76f..426c9bd 100644 --- a/script/get-dataset-cognata-mlcommons/customize.py +++ b/script/get-dataset-cognata-mlcommons/customize.py @@ -1,32 +1,295 @@ from cmind import utils import os - +import json def preprocess(i): env = i['env'] - script_path = i['run_script_input']['path'] + automation = i['automation'] + cm = automation.cmind + + cur_dir = os.getcwd() + + quiet = (env.get('CM_QUIET', False) == 'yes') + + dataset_path = env.get('CM_CUSTOM_CACHE_ENTRY_MLCOMMONS_COGNATA_DATASET_PATH','') + + # First level dir + dataset_path1 = os.path.join(dataset_path, 'data') + + if not os.path.isdir(dataset_path1): + os.makedirs(dataset_path1) + + # Check if has license and download URL + dataset_path_secret = os.path.join(dataset_path1, 'secret.json') + + first_url = '' + dataset_meta = {} + + if os.path.isfile(dataset_path_secret): + r = utils.load_json(dataset_path_secret) + if r['return']>0: return r + + dataset_meta = r['meta'] + + first_url = dataset_meta.get('first_url','').strip() + + if first_url == '': + print ('') + first_url = input ('Please register at https://mlcommons.org/datasets/cognata and enter private URL: ') + + first_url = first_url.strip() + + if first_url == '': + return {'return':1, 'error': 'Private MLCommons Cognata URL was not provided'} + + dataset_meta['first_url'] = first_url + + with open (dataset_path_secret, 'w') as f: + f.write(json.dumps(dataset_meta, indent=2)+'\n') + + ############################################################################# + # Check if first.xlsx exists + file_first_xlsx = 'first.xlsx' + first_xlsx = os.path.join(dataset_path1, file_first_xlsx) + + if not os.path.isfile(first_xlsx): + # Attempting to download file + first_url_export, dummy = google_url_for_export(first_url) + + if first_url_export == '': + return {'return':1, 'error':'can\'t parse URL for export: {}'.format(first_url)} + + r = cm.access({'action':'run', + 'automation':'script', + 'tags':'download,file,_wget', + 'verify':'no', + 'url':first_url_export, + 'output_file': file_first_xlsx, + 'store': dataset_path1}) + if r['return']>0: return r + + if not os.path.isfile(first_xlsx): + return {'return':1, 'error':'File {} was not downloaded'.format(first_xlsx)} + + ############################################################################# + # Parse XLSX and check serial number + serial_numbers = [] + for s in env.get('CM_MLCOMMONS_COGNATA_SERIAL_NUMBERS','').strip().split(','): + s=s.strip() + if s!='' and s not in serial_numbers: + serial_numbers.append(s) + + + dataset_key = env['CM_MLCOMMONS_COGNATA_KEY1'] + url_key = 'Link to Excel File (Download Links)' + serial_key = 'Serial Number' + + r = process_xlsx(first_xlsx, dataset_key, url_key, serial_key, serial_numbers) + if r['return']>0: return r + + headers = r['headers'] + data = r['data'] + all_data = r['all_data'] + + if len(all_data)!=0: + file_first_json = 'first.json' + first_json = os.path.join(dataset_path1, file_first_json) + + if not os.path.isfile(first_json): + with open (first_json, 'w') as f: + f.write(json.dumps(all_data, indent=2)+'\n') + + if len(data)==0: + return {'return':0, 'error': 'no sets found'} + + ############################################################################# + print ('') + print ('Available or selected serial numbers (use --serial_numbers=a,b,c to download specific subsets):') + print ('') + for d in data: + s = d[serial_key] + print (s) + + for d in data: + url = d[url_key] + url_export, dummy = google_url_for_export(url) + + serial_file = d[serial_key] + '.xlsx' + + dataset_path2 = os.path.join(dataset_path1, serial_file) + dataset_path3 = os.path.join(dataset_path1, d[serial_key]) + + if not os.path.isdir(dataset_path3): + os.makedirs(dataset_path3) + + if not os.path.isfile(dataset_path2): + + print ('') + print ('Downloading {} ...'.format(url_export)) + + r = cm.access({'action':'run', + 'automation':'script', + 'tags':'download,file,_wget', + 'verify':'no', + 'url':url_export, + 'output_file': serial_file, + 'store': dataset_path1}) + if r['return']>0: return r + + ############################################################################# + print ('') + print ('Processing subsets ...') + + group_names = [] + for s in env.get('CM_MLCOMMONS_COGNATA_GROUP_NAMES','').strip().split(','): + s=s.strip() + if s!='' and s not in group_names: + group_names.append(s) + + + for d in data: + serial_file = d[serial_key] + '.xlsx' - input('xyz11') + dataset_path2 = os.path.join(dataset_path1, serial_file) + dataset_path3 = os.path.join(dataset_path1, d[serial_key]) + + print ('') + print ('Processing {} ...'.format(serial_file)) + dataset_key = 'File_Data' + url_key = 'File_Link' + serial_key = 'Group_Name' + + r = process_xlsx(dataset_path2, dataset_key, url_key, serial_key, group_names) + if r['return']>0: return r + + headers = r['headers'] + data = r['data'] + all_data = r['all_data'] + + if len(all_data)!=0: + file_all_json = 'all.json' + all_json = os.path.join(dataset_path3, file_all_json) + + if not os.path.isfile(all_json): + with open (all_json, 'w') as f: + f.write(json.dumps(all_data, indent=2)+'\n') + + if len(data)==0: + return {'return':0, 'error': 'no sub-sets found'} + + for d in data: + file_name = d['File_Name'] + file_name_with_path = os.path.join(dataset_path3, file_name) + file_name_with_path_done = os.path.join(dataset_path3, file_name)+'.done' + + url = d[url_key] + + print ('') + print ('Downloading {} ...'.format(file_name)) + + if os.path.isfile(file_name_with_path_done): + print ('') + print (' Already processed - skipping ...') + continue + + if os.path.isfile(file_name_with_path): + os.remove(file_name_with_path) + + if os.name == 'nt': + aria2_tool = env['CM_ARIA2_BIN_WITH_PATH'] + else: + aria2_tool = 'aria2c' + + + cmd = aria2_tool + ' --async-dns=false -x15 -s15 "{}" --dir "{}" -o "{}"'.format(url, dataset_path3, file_name) + + print ('') + print (cmd) + print ('') + + os.system(cmd) + + # Unzip + if file_name.endswith('.zip'): + + print ('') + print ('Extracting file {} ...'.format(file_name_with_path)) + print ('') + r = cm.access({'action':'run', + 'automation':'script', + 'tags':'extract,file', + 'input':file_name_with_path, + 'extract_path': dataset_path3}) + if r['return']>0: return r + + # Mark as downloaded + with open(file_name_with_path_done, 'w') as f: + f.write('DONE\n') + + print ('') return {'return': 0} + + + + def postprocess(i): env = i['env'] - input('xyz22') + return {'return': 0} + +# Prepare Google URL for export +def google_url_for_export(url): + url2 = '' + + j = url.rfind('/') + + if j>0: + url = url[:j+1] + url2 = url + 'export' + + return (url2, url) + +# Download Cognata XLSX +def process_xlsx(filename, dataset_key, url_key, serial_key, serial_numbers): + import openpyxl - return {'return': 1, 'error':'TBD'} #todo + ex = openpyxl.load_workbook(filename) + + sets = ex[dataset_key] + headers = {} + + data = [] + all_data = [] - if env.get('CM_DATASET_CALIBRATION','') == "no": - env['CM_DATASET_PATH_ROOT'] = os.path.join(os.getcwd(), 'install') - env['CM_DATASET_PATH'] = os.path.join(os.getcwd(), 'install', 'validation', 'data') - env['CM_DATASET_CAPTIONS_DIR_PATH'] = os.path.join(os.getcwd(), 'install', 'captions') - env['CM_DATASET_LATENTS_DIR_PATH'] = os.path.join(os.getcwd(), 'install', 'latents') - else: - env['CM_CALIBRATION_DATASET_PATH'] = os.path.join(os.getcwd(), 'install', 'calibration', 'data') + for row in sets.iter_rows(values_only=True): + lrow = list(row) - return {'return': 0} + if len(headers) == 0: + for j in range(0,len(lrow)): + headers[j]=str(lrow[j]).strip() + else: + xrow = {} + + for j in range(0, len(lrow)): + xrow[headers[j]] = lrow[j] + + url = str(xrow.get(url_key, '')) + if 'https' in url: + all_data.append(xrow) + + if len(serial_numbers)>0: + serial_number = xrow.get(serial_key, '') + + if serial_number not in serial_numbers: + continue + + if url!='': + data.append(xrow) + + return {'return':0, 'headers':headers, 'data':data, 'all_data':all_data} diff --git a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml index 5b02fc5..b3bb962 100644 --- a/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml +++ b/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml @@ -37,6 +37,7 @@ deps: - python - python3 +######################################################################## - tags: get,generic-python-lib,_numpy - tags: get,generic-python-lib,_package.typing-extensions - tags: get,generic-python-lib,_package.sympy @@ -85,7 +86,7 @@ deps: - +############################################################ variations: cpu: group: device @@ -102,6 +103,7 @@ variations: env: CM_DEVICE: cuda +############################################################ input_description: input: desc: input image (png) @@ -113,6 +115,7 @@ input_description: desc: change number of classes (default = 13) +############################################################ docker: skip_run_cmd: 'no' gh_token_required: True From 4584c2ca39ae4a984efa6d91d7fa83f8e033dc68 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 19 Apr 2024 18:24:18 +0200 Subject: [PATCH 32/33] update --- docs/test-abtf-model/README.md | 63 +++++++++++++++---- .../README-extra.md | 6 +- 2 files changed, 57 insertions(+), 12 deletions(-) diff --git a/docs/test-abtf-model/README.md b/docs/test-abtf-model/README.md index 2d603fc..2ad7807 100644 --- a/docs/test-abtf-model/README.md +++ b/docs/test-abtf-model/README.md @@ -37,7 +37,7 @@ call ABTF\Scripts\activate.bat & set CM_REPOS=%CD%\ABTF\CM & cd ABTF\work ## Install CM automations -### CM for MLOps +### CM for MLOps and DevOps ```bash cm pull repo mlcommons@cm4mlops --checkout=dev @@ -55,7 +55,7 @@ cm pull repo mlcommons@cm4abtf --checkout=dev cm show repo ``` -### Fins a specific repo +### Find a specific repo ```bash cm find repo mlcommons@cm4abtf @@ -69,17 +69,18 @@ cm pull repo ``` -### Download ABTF model and sample image +### Download ABTF models and sample image ```bash -cmr "download file _wget" --url="https://www.dropbox.com/scl/fi/od48qvnbqyfuy1z3aas84/baseline_8mp_ss_scales_ep15.pth?rlkey=d6ybe7g09g21pondmbd3pivzk&dl=0" --verify=no --md5sum=c36cb56b5f6bf8edbe64f9914506e09d -cmr "download file _wget" --url="https://www.dropbox.com/scl/fi/0n7rmxxwqvg04sxk7bbum/0000008766.png?rlkey=mhmr3ztrlsqk8oa67qtxoowuh&dl=0" --verify=no --md5sum=903306a7c8bfbe6c1ca68fad6e34fe52 +cmr "download file _wget" --url="https://www.dropbox.com/scl/fi/9un2i2169rgebui4xklnm/baseline_8MP_ss_scales_all_ep60.pth?rlkey=sez3dnjep4waa09s5uy4r3wmk&st=z859czgk&dl=0" --verify=no --md5sum=1ab66f523715f9564603626e94e59c8c +cmr "download file _wget" --url="https://www.dropbox.com/scl/fi/od48qvnbqyfuy1z3aas84/baseline_8mp_ss_scales_ep15.pth?rlkey=d6ybe7g09g21pondmbd3pivzk&dl=0" --verify=no --md5sum=c36cb56b5f6bf8edbe64f9914506e09d -s +cmr "download file _wget" --url="https://www.dropbox.com/scl/fi/0n7rmxxwqvg04sxk7bbum/0000008766.png?rlkey=mhmr3ztrlsqk8oa67qtxoowuh&dl=0" --verify=no --md5sum=903306a7c8bfbe6c1ca68fad6e34fe52 -s ``` ### Download ABTF model code and register in CM cache ```bash -cmr "get ml-model abtf-ssd-pytorch _local.baseline_8mp_ss_scales_ep15.pth" +cmr "get ml-model abtf-ssd-pytorch" ``` ### Check the state of the CM cache @@ -101,30 +102,56 @@ Next commands prepare environment to benchmark host CPU. Check these docs to benchmark other devices: * [CUDA-based device](README-cuda.md) -### Build MLPerf loadgen + +### Detect python from virtual env ```bash -cmr "get mlperf inference loadgen _copy" --version=main +cmr "get python3" --quiet ``` ### Install or detect PyTorch + ```bash cmr "get generic-python-lib _torch" cmr "get generic-python-lib _torchvision" ``` +If you want to install a specific version of PyTorch, you can specify it as follows: + +```bash +cmr "get generic-python-lib _torch" --version=2.2.0 +cmr "get generic-python-lib _torchvision" --version=0.17.0 +``` + + +## Run ABTF Model with a test image and prepare for loadgen + +```bash +cmr "test abtf ssd-resnet50 cognata pytorch" --model=baseline_8MP_ss_scales_all_ep60.pth --config=baseline_8MP_ss_scales_all --input=0000008766.png --output=0000008766_prediction_test.jpg +cmr "test abtf ssd-resnet50 cognata pytorch" --model=baseline_8mp_ss_scales_ep15.pth --config=baseline_8MP_ss_scales --input=0000008766.png --output=0000008766_prediction_test.jpg --num-classes=13 +``` + +CM will load a workflow described by [this simple YAML](https://github.com/mlcommons/cm4abtf/blob/dev/script/test-ssd-resnet50-cognata-pytorch/_cm.yaml), +call other CM scripts to detect or build missing deps for a given platform, prepare all environment variables and run benchmark. +You can run it in silent mode to skip CM workflow information using `-s` or `--silent` flag: +```bash +cmr "test abtf ssd-resnet50 cognata pytorch" --model=baseline_8MP_ss_scales_all_ep60.pth --config=baseline_8MP_ss_scales_all --input=0000008766.png --output=0000008766_prediction_test.jpg -s +``` -## Test Model with a test image +## Benchmark performance of ABTF model with MLPerf loadgen + +### Build MLPerf loadgen ```bash -cmr "test abtf ssd-resnet50 cognata pytorch" --input=0000008766.png --output=0000008766_prediction_test.jpg --config=baseline_8MP_ss_scales --num-classes=13 +cmr "get mlperf inference loadgen _copy" --version=main ``` -## Benchmark model with MLPerf loadgen +### Run ABTF model with loadgen ```bash +cmr "generic loadgen python _pytorch _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cpu.pickle --modelpath=baseline_8MP_ss_scales_all_ep60.pth --modelcfg.config=baseline_8MP_ss_scales_all cmr "generic loadgen python _pytorch _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cpu.pickle --modelpath=baseline_8mp_ss_scales_ep15.pth --modelcfg.num_classes=13 --modelcfg.config=baseline_8MP_ss_scales ``` @@ -151,6 +178,20 @@ cmr "generic loadgen python _pytorch _custom _cmc" --samples=5 --modelsamplepath ``` +## Benchmark accuracy of ABTF model with MLPerf loadgen + +### Download Cognata data set via CM + +We have developed a [CM automation recipe](https://github.com/mlcommons/cm4abtf/blob/dev/script/get-dataset-cognata-mlcommons/customize.py) +to download different sub-sets of the MLCommons Cognata data set - you just need to provide a private URL after registering to access +this dataset [here](https://mlcommons.org/datasets/cognata/): + + +```bash +cmr "get raw dataset mlcommons-cognata" --serial_numbers=10002_Urban_Clear_Morning --group_names=Cognata_Camera_02_8M +``` + + diff --git a/script/get-dataset-cognata-mlcommons/README-extra.md b/script/get-dataset-cognata-mlcommons/README-extra.md index 3edbe8c..db8ab69 100644 --- a/script/get-dataset-cognata-mlcommons/README-extra.md +++ b/script/get-dataset-cognata-mlcommons/README-extra.md @@ -1,2 +1,6 @@ -cmr "get raw dataset mlcommons-cognata" --serial_numbers=10002_Urban_Clear_Morning +Examples: + +```bash +cmr "get raw dataset mlcommons-cognata" --serial_numbers=10002_Urban_Clear_Morning cmr "get raw dataset mlcommons-cognata" --serial_numbers=10002_Urban_Clear_Morning --group_names=Cognata_Camera_02_8M +``` From 932166b17dfdf9356dae3410b84749e611a45951 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 19 Apr 2024 18:27:20 +0200 Subject: [PATCH 33/33] clean up --- docs/test-abtf-model/README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/test-abtf-model/README.md b/docs/test-abtf-model/README.md index 2ad7807..2a8fcef 100644 --- a/docs/test-abtf-model/README.md +++ b/docs/test-abtf-model/README.md @@ -103,6 +103,8 @@ Check these docs to benchmark other devices: * [CUDA-based device](README-cuda.md) + + ### Detect python from virtual env ```bash @@ -151,7 +153,14 @@ cmr "get mlperf inference loadgen _copy" --version=main ### Run ABTF model with loadgen ```bash -cmr "generic loadgen python _pytorch _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cpu.pickle --modelpath=baseline_8MP_ss_scales_all_ep60.pth --modelcfg.config=baseline_8MP_ss_scales_all +cmr "test abtf ssd-resnet50 cognata pytorch" --model=baseline_8MP_ss_scales_all_ep60.pth --config=baseline_8MP_ss_scales_all --input=0000008766.png --output=0000008766_prediction_test.jpg +cmr "generic loadgen python _pytorch _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cpu.pickle --modelpath=baseline_8MP_ss_scales_all_ep60.pth --modelcfg.num_classes=15 --modelcfg.config=baseline_8MP_ss_scales_all +``` + +or older version + +``` +cmr "test abtf ssd-resnet50 cognata pytorch" --model=baseline_8mp_ss_scales_ep15.pth --config=baseline_8MP_ss_scales --input=0000008766.png --output=0000008766_prediction_test.jpg --num-classes=13 cmr "generic loadgen python _pytorch _custom _cmc" --samples=5 --modelsamplepath=0000008766.png.cpu.pickle --modelpath=baseline_8mp_ss_scales_ep15.pth --modelcfg.num_classes=13 --modelcfg.config=baseline_8MP_ss_scales ```