Merge pull request #29 from mlcommons/dev

Added Docker container generation and debug script to run without CM

CHANGES.md

@@ -1,6 +1,14 @@
 ### 20240429
  * Added `cm set cfg` automation. For example, we can set default CM script to silent
    using `cm set cfg default --key.script.silent`
+ * Added key `script_entry_repo_to_report_errors` to CM script meta
+   to customize where to report errors instead of the default repo.
+   For example, issues with the CM script `run-mlperf-inference-app` 
+   should be reported at https://github.com/mlcommons/inference .
+ * Added saving running script from different deps without CM to tmp-run-without-cm.bat.
+   Example: `cmr "app image corner-detection" --debug-script-tags=benchmark,program`
+ * Generate Docker container sample during --repro (prototype)
+
 
 ### 20240427
  * Added cm run script --print_versions to print resolved versions of dependencies at the end of the run

automation/script/docker_repro_example/ubuntu-23.04.Dockerfile

@@ -0,0 +1,34 @@
+FROM ubuntu:23.04
+
+# Automatically generated by the CM workflow automation meta-framework
+# https://github.com/mlcommons/ck
+
+LABEL github=""
+LABEL maintainer=""
+LABEL license=""
+
+SHELL ["/bin/bash", "-c"]
+
+ARG CM_GH_TOKEN
+ARG CM_ADD_DOCKER_GROUP_ID=""
+
+# Notes: https://runnable.com/blog/9-common-dockerfile-mistakes
+# Install system dependencies
+RUN apt-get update -y
+RUN apt-get install -y python3 python3-pip git sudo wget
+
+# Setup docker environment
+ENTRYPOINT ["/bin/bash", "-c"]
+ENV TZ="US/Pacific"
+ENV PATH="${PATH}:/home/cmuser/.local/bin"
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone
+
+# Setup docker user
+RUN groupadd ${CM_ADD_DOCKER_GROUP_ID} cm
+RUN useradd  -g cm --create-home --shell /bin/bash cmuser
+RUN echo "cmuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+USER cmuser:cm
+WORKDIR /home/cmuser
+
+# Install python packages
+RUN python3 -m pip install --user cmind requests giturlparse tabulate  --break-system-packages 

automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.build.bat

@@ -0,0 +1 @@
+docker build  -f "ubuntu-23.04.Dockerfile" -t "cknowledge/test-cm-script:ubuntu-23.04-cm-dev" .

automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker build  -f "ubuntu-23.04.Dockerfile" -t "cknowledge/test-cm-script:ubuntu-23.04-cm-dev" .

automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.run.bat

@@ -0,0 +1 @@
+docker run  -it --entrypoint ""  cknowledge/test-cm-script:ubuntu-23.04-cm-dev bash

automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.run.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker run  -it --entrypoint ""  cknowledge/test-cm-script:ubuntu-23.04-cm-dev bash

automation/script/module.py

@@ -775,6 +775,12 @@ def _run(self, i):
         run_state['script_tags'] = script_tags
         run_state['script_variation_tags'] = variation_tags
         run_state['script_repo_alias'] = script_artifact.repo_meta.get('alias', '')
+        run_state['script_repo_git'] = script_artifact.repo_meta.get('git', False)
+
+        if not recursion:
+            run_state['script_entry_repo_to_report_errors'] = meta.get('repo_to_report_errors', '')
+            run_state['script_entry_repo_alias'] = script_artifact.repo_meta.get('alias', '')
+            run_state['script_entry_repo_git'] = script_artifact.repo_meta.get('git', False)
 
         deps = meta.get('deps',[])
         post_deps = meta.get('post_deps',[])
@@ -1794,6 +1800,23 @@ def _run(self, i):
         if print_readme or repro_prefix!='':
             readme = self._get_readme(cmd, run_state)
 
+        # Copy Docker sample
+        if repro_prefix!='' and repro_dir!='':
+            docker_template_path = os.path.join(self.path, 'docker_repro_example')
+            if os.path.isdir(docker_template_path):
+                try:
+                   shutil.copytree(docker_template_path, repro_dir, dirs_exist_ok=True)
+                except Exception as e:
+                   pass
+
+            docker_container = self._get_docker_container(cmd, run_state)
+
+            try:
+               with open (os.path.join(repro_dir, 'ubuntu-23.04.Dockerfile'), 'a+') as f:
+                  f.write(docker_container)
+            except:
+               pass
+
         if print_readme:
             with open('README-cm.md', 'w') as f:
                 f.write(readme)
@@ -3124,6 +3147,58 @@ def _get_readme(self, cmd_parts, run_state):
 
         return content
 
+    ##############################################################################
+    def _get_docker_container(self, cmd_parts, run_state):
+        """
+        Outputs a Markdown README file listing the CM run commands for the dependencies
+        """
+
+        deps = run_state['deps']
+
+        version_info = run_state.get('version_info', [])
+        version_info_dict = {}
+
+        for v in version_info:
+            k = list(v.keys())[0]
+            version_info_dict[k]=v[k]
+
+        content = ''
+
+        content += """
+
+# The following CM commands were automatically generated (prototype)
+
+cm pull repo mlcommons@cm4mlops --checkout=dev
+
+"""
+        current_cm_repo = run_state['script_repo_alias']
+        if current_cm_repo not in ['mlcommons@ck', 'mlcommons@cm4mlops']:
+            content += '\ncm pull repo ' + run_state['script_repo_alias'] + '\n\n'
+
+
+        deps_ = ''
+
+        for dep_tags in deps:
+
+            xversion = ''
+            version = version_info_dict.get(dep_tags, {}).get('version','')
+            if version !='' :
+                xversion = ' --version={}\n'.format(version)
+
+            content += "# cm run script --tags=" + dep_tags + "{}\n\n".format(xversion)
+
+        cmd="cm run script "
+
+        for cmd_part in cmd_parts:
+            x = '"' if ' ' in cmd_part and not cmd_part.startswith('-') else ''
+            cmd = cmd + " " + x + cmd_part + x
+
+        content += cmd + '\n'
+
+
+        return content
+
+
     ##############################################################################
     def _print_versions(self, run_state):
         """
@@ -4467,6 +4542,7 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"):
             os.remove(tmp_file_run_env)
 
         run_script = tmp_file_run + bat_ext
+        run_script_without_cm = tmp_file_run + '-without-cm' + bat_ext
 
         if verbose:
             print ('')
@@ -4490,16 +4566,17 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"):
 
         script += convert_env_to_script(env, os_info)
 
-        # Check if run bash/cmd before running the command (for debugging)
-        if debug_script_tags !='' and all(item in found_script_tags for item in debug_script_tags.split(',')):
-            x=['cmd', '.', '','.bat'] if os_info['platform'] == 'windows' else ['bash', ' ""', '"','.sh']
-
-            script.append('\n')
-            script.append('echo{}\n'.format(x[1]))
-            script.append('echo {}Running debug shell. Type exit to resume script execution ...{}\n'.format(x[2],x[3],x[2]))
-            script.append('echo{}\n'.format(x[1]))
-            script.append('\n')
-            script.append(x[0])
+#        # Check if run bash/cmd before running the command (for debugging)
+#        if debug_script_tags !='' and all(item in found_script_tags for item in debug_script_tags.split(',')):
+#            # Copy original run script to be able to run it outside ...
+#            x=['cmd', '.', '','.bat'] if os_info['platform'] == 'windows' else ['bash', ' ""', '"','.sh']
+#
+#            script.append('\n')
+#            script.append('echo{}\n'.format(x[1]))
+#            script.append('echo {}Running debug shell. Type exit to resume script execution ...{}\n'.format(x[2],x[2]))
+#            script.append('echo{}\n'.format(x[1]))
+#            script.append('\n')
+#            script.append(x[0])
 
         # Append batch file to the tmp script
         script.append('\n')
@@ -4509,6 +4586,16 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"):
         r = record_script(run_script, script, os_info)
         if r['return']>0: return r
 
+        # Save file to run without CM
+        if debug_script_tags !='' and all(item in found_script_tags for item in debug_script_tags.split(',')):
+
+            import shutil
+            shutil.copy(run_script, run_script_without_cm)
+
+            print ('================================================================================')
+            print ('Debug script to run without CM was recorded: {}'.format(run_script_without_cm))
+            print ('================================================================================')
+
         # Run final command
         cmd = os_info['run_local_bat_from_python'].replace('${bat_file}', run_script)
 
@@ -4528,14 +4615,32 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"):
                            print (r['string'])
                            print ("")
 
+
+            # Check where to report errors and failures
+            repo_to_report = run_state.get('script_entry_repo_to_report_errors', '')
+
+            if repo_to_report == '':
+                script_repo_alias = run_state.get('script_repo_alias', '')
+                script_repo_git = run_state.get('script_repo_git', False)
+
+                if script_repo_git and script_repo_alias!='':
+                    repo_to_report = 'https://github.com/'+script_repo_alias.replace('@','/')+'/issues'
+
+            if repo_to_report == '':
+                repo_to_report = 'https://github.com/mlcommons/ck/issues'
+
             note = '''
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Note that it may be a portability issue of a third-party tool or a native script 
-wrapped and unified by this automation recipe (CM script). In such case, 
-please report this issue with a full log at "https://github.com/mlcommons/ck". 
+Note that it is often a portability issue of a third-party tool or a native script 
+wrapped and unified by this CM script (automation recipe). Please re-run
+this script with --repro flag and report this issue with the original
+command line, cm-repro directory and full log here:
+
+{}
+
 The CM concept is to collaboratively fix such issues inside portable CM scripts 
 to make existing tools and native scripts more portable, interoperable 
-and deterministic. Thank you'''
+and deterministic. Thank you'''.format(repo_to_report)
 
             rr = {'return':2, 'error':'Portable CM script failed (name = {}, return code = {})\n\n{}'.format(meta['alias'], rc, note)}
 

script/app-image-corner-detection/README-extra.md

@@ -3,10 +3,10 @@
 First download images:
 
 ```bash
-cmr "download file _wget" --url=https://cKnowledge.org/ai/data/data.pgm --verify=no --env.CM_DOWNLOAD_CHECKSUM=0af279e557a8de252d7ff0751a999379
-cmr "download file _wget" --url=https://cKnowledge.org/ai/data/computer_mouse.jpg --verify=no --env.CM_DOWNLOAD_CHECKSUM=45ae5c940233892c2f860efdf0b66e7e
-cmr "download file _wget" --url=https://cKnowledge.org/ai/data/computer_mouse2.jpg --verify=no --env.CM_DOWNLOAD_CHECKSUM=e7e2050b41e0b85cedca3ca87ab55390
-cmr "download file _wget" --url=https://cKnowledge.org/ai/data/computer_mouse2.pgm --verify=no --env.CM_DOWNLOAD_CHECKSUM=a4e48556d3eb09402bfc98e375b41311
+cmr "download file _wget" --url=https://cKnowledge.org/ai/data/data.pgm --ssl-verify=no --md5sum=0af279e557a8de252d7ff0751a999379
+cmr "download file _wget" --url=https://cKnowledge.org/ai/data/computer_mouse.jpg --ssl-verify=no --md5sum=45ae5c940233892c2f860efdf0b66e7e
+cmr "download file _wget" --url=https://cKnowledge.org/ai/data/computer_mouse2.jpg --ssl-verify=no --md5sum=e7e2050b41e0b85cedca3ca87ab55390
+cmr "download file _wget" --url=https://cKnowledge.org/ai/data/computer_mouse2.pgm --ssl-verify=no --md5sum=a4e48556d3eb09402bfc98e375b41311
 ```
 
 Then run app
@@ -23,3 +23,10 @@ cm run script "app image corner-detection" -add_deps_recursive.compiler.tags=llv
 * Ubuntu 22.04; x64; LLVM 17.06
 * Windows 11; x64; LLVM 17.06
 
+## Debugging scripts without CM
+
+```bash
+cmr "app image corner-detection" --debug_script_tags=compile,cpp-program
+cmr "app image corner-detection" --debug-script-tags=benchmark,program
+```
+

script/app-image-corner-detection/_cm.yaml

@@ -0,0 +1,32 @@
+alias: app-image-corner-detection
+uid: 998ffee0bc534d0a
+
+automation_alias: script
+automation_uid: 5b4e0237da074764
+
+category: Modular application pipeline
+
+deps:
+- tags: detect,os
+
+- tags: detect,cpu
+
+- tags: download,file,_url.https://cKnowledge.org/ai/data/data.pgm
+  md5sum: 0af279e557a8de252d7ff0751a999379
+  force_cache: false
+
+
+posthook_deps:
+- skip_if_env:
+    CM_SKIP_COMPILE:
+    - 'on'
+  tags: compile,cpp-program
+- skip_if_env:
+    CM_SKIP_RUN:
+    - 'on'
+  tags: benchmark-program
+
+tags:
+- app
+- image
+- corner-detection

script/app-image-corner-detection/customize.py

@@ -12,10 +12,17 @@ def preprocess(i):
 
     if 'CM_INPUT' not in env:
         env['CM_INPUT'] = os.path.join(script_path, 'data.pgm')
+
     if 'CM_OUTPUT' not in env:
         env['CM_OUTPUT'] = 'output_image_with_corners.pgm'
+
     if 'CM_RUN_DIR' not in env:
-        env['CM_RUN_DIR'] = os.path.join(script_path, "output")
+        output_path = os.path.join(script_path, "output")
+        if output_path!='' and not os.path.isdir(output_path):
+            os.makedirs(output_path)
+
+        env['CM_RUN_DIR'] = output_path
+
     env['CM_RUN_SUFFIX']= env['CM_INPUT'] + ' ' + env['CM_OUTPUT'] + ' -c'
 
     if os_info['platform'] == 'windows':

script/run-mlperf-inference-app/_cm.yaml

@@ -499,3 +499,4 @@ input_description:
     boolean: true
     default: false
 
+#repo_to_report_errors: https://github.com/mlcommons/inference/issues