From b3c0005930a55d4dcb034d7fc7f960bfcd2122b8 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Sun, 8 Dec 2024 23:12:41 +0100 Subject: [PATCH 1/3] formatted Python modules from the internal repository using autopep8 --- README.md | 30 +- cm-mlops/README.md | 2 +- cm/CHANGES.md | 3 + cm/cmind/repo/automation/automation/module.py | 76 +- .../repo/automation/automation/module_misc.py | 149 ++-- cm/cmind/repo/automation/ckx/module.py | 15 +- cm/cmind/repo/automation/core/module.py | 120 +-- cm/cmind/repo/automation/core/module_misc.py | 4 +- cm/cmind/repo/automation/repo/module.py | 833 +++++++++--------- 9 files changed, 650 insertions(+), 582 deletions(-) diff --git a/README.md b/README.md index e5b595b3a..164faed25 100755 --- a/README.md +++ b/README.md @@ -13,28 +13,31 @@ [Collective Knowledge (CK, CM, CM4MLOps, CM4MLPerf and CMX)](https://cKnowledge.org) is an educational community project to learn how to run AI, ML and other emerging workloads -in the most efficient and cost-effective way across diverse models, data sets, software and hardware. +in the most efficient and cost-effective way across diverse models, data sets, software and hardware: +[ [white paper](https://arxiv.org/abs/2406.16791) ]. + It includes the following sub-projects. ### Collective Mind (CM) -[Collective Mind (CM)](https://github.com/mlcommons/ck/tree/master/cm) - -a very lightweight Python-based framework with a unified CLI, Python API and minimal dependencies -intended to help researchers and engineers automate their repetitive, tedious and time-consuming tasks -to build, run, benchmark and optimize AI, ML and other applications and systems -across diverse and continuously changing models, data, software and hardware. +The [Collective Mind framework](https://github.com/mlcommons/ck/tree/master/cm) +is a lightweight, Python-based toolset featuring +a unified command-line interface (CLI), Python API, and minimal dependencies. +It is designed to assist researchers and engineers in automating repetitive, time-consuming +tasks such as building, running, benchmarking, and optimizing AI, machine learning, +and other applications across diverse and continuously changing models, data, software and hardware. -Collective Mind is continuously enhanced through public and private CM4* Git repositories, -which serve as the unified interface for various collections of reusable automations and artifacts. +Collective Mind is continuously enhanced through public and private Git repositories +with CM automation recipes and artifacts accessible via unified CM interface. The CM architecture diagram is available for viewing [here](https://github.com/mlcommons/ck/tree/master/docs/specs/cm-diagram-v3.5.1.png). -### Collective Mind repositories +### Notable Collective Mind repositories #### CM4MLOps -[CM4MLOPS repository powered by CM](https://github.com/mlcommons/cm4mlops) - +[CM4MLOPS repository powered by CM](https://github.com/mlcommons/ck/tree/master/cm-mlops) - a collection of portable, extensible and technology-agnostic automation recipes with a common CLI and Python API (CM scripts) to unify and automate all the manual steps required to compose, run, benchmark and optimize complex ML/AI applications @@ -121,13 +124,6 @@ based on user feedback. Follow the project's progress [here]( https://github.com * [Grigori Fursin](https://cKnowledge.org/gfursin) (FlexAI, cTuning) -## Maintainers - -* Collective Mind (CM): [Grigori Fursin](https://cKnowledge.org/gfursin) -* CM4MLOps repository: [Arjun Suresh](https://github.com/arjunsuresh) and [Anandhu Sooraj](https://github.com/anandhu-eng) -* CMX (the next generation of CM and CM4MLOps): [Grigori Fursin](https://cKnowledge.org/gfursin) - - ## Citing Collective Mind and Collective Knowledge If you found the CM automation framework helpful, kindly reference this article: diff --git a/cm-mlops/README.md b/cm-mlops/README.md index f65fdc267..19062d827 100644 --- a/cm-mlops/README.md +++ b/cm-mlops/README.md @@ -1,5 +1,5 @@ **This repository is archived. In April 2024, we have separated this CM repo from the CM framework - and moved it to a [standalone mlcommons@cm4mlops repository](https://github.com/mlcommons/cm4mlops/tree/dev) + and moved it to a [standalone mlcommons@cm4mlops repository](https://github.com/mlcommons/cm4mlops) following the suggesion from our users and MLCommons members.** We keep this directory for backwards compatibility to let users reproduce past projects relying on this repository. diff --git a/cm/CHANGES.md b/cm/CHANGES.md index ba2f4fd4f..698ef7493 100644 --- a/cm/CHANGES.md +++ b/cm/CHANGES.md @@ -1,3 +1,6 @@ +## V3.5.1.1 + - formatted Python modules from the internal repository using autopep8 + ## V3.5.1 - added Collective Mind architecture diagram: https://github.com/mlcommons/ck/tree/master/docs/specs/cm-diagram-v3.5.1.png diff --git a/cm/cmind/repo/automation/automation/module.py b/cm/cmind/repo/automation/automation/module.py index 66fe9bb78..d1551df31 100644 --- a/cm/cmind/repo/automation/automation/module.py +++ b/cm/cmind/repo/automation/automation/module.py @@ -7,6 +7,7 @@ from cmind.automation import Automation from cmind import utils + class CAutomation(Automation): """ CM "automation" automation actions @@ -35,12 +36,12 @@ def print_input(self, i): """ import json - print (json.dumps(i, indent=2)) - - return {'return':0} + print(json.dumps(i, indent=2)) + return {'return': 0} ############################################################ + def add(self, i): """ Add CM automation. @@ -66,36 +67,40 @@ def add(self, i): console = i.get('out') == 'con' - parsed_artifact = i.get('parsed_artifact',[]) + parsed_artifact = i.get('parsed_artifact', []) - artifact_obj = parsed_artifact[0] if len(parsed_artifact)>0 else ('','') + artifact_obj = parsed_artifact[0] if len( + parsed_artifact) > 0 else ('', '') module_name = 'module.py' tags_list = utils.convert_tags_to_list(i) - if 'automation' not in tags_list: tags_list.append('automation') + if 'automation' not in tags_list: + tags_list.append('automation') # Add placeholder (use common action) - i['out']='con' - i['common']=True + i['out'] = 'con' + i['common'] = True - i['meta']={'automation_alias':self.meta['alias'], - 'automation_uid':self.meta['uid'], - 'tags':tags_list} + i['meta'] = {'automation_alias': self.meta['alias'], + 'automation_uid': self.meta['uid'], + 'tags': tags_list} - if 'tags' in i: del(i['tags']) + if 'tags' in i: + del (i['tags']) automation = i['automation'] - if automation!='.' and ',' not in automation: + if automation != '.' and ',' not in automation: i['automation'] = automation + ',' + self.meta['uid'] - r_obj=self.cmind.access(i) - if r_obj['return']>0: return r_obj + r_obj = self.cmind.access(i) + if r_obj['return'] > 0: + return r_obj new_automation_path = r_obj['path'] if console: - print ('Created automation in {}'.format(new_automation_path)) + print('Created automation in {}'.format(new_automation_path)) # Create Python module holder module_holder_path = new_automation_path @@ -106,12 +111,12 @@ def add(self, i): # Copy module files for f in ['module_dummy.py']: f1 = os.path.join(self.path, f) - f2 = os.path.join(new_automation_path, f.replace('_dummy','')) + f2 = os.path.join(new_automation_path, f.replace('_dummy', '')) if console: - print (' * Copying {} to {}'.format(f1, f2)) + print(' * Copying {} to {}'.format(f1, f2)) - shutil.copyfile(f1,f2) + shutil.copyfile(f1, f2) return r_obj @@ -137,20 +142,23 @@ def add_cmx(self, i): # Prepare to call common function r = utils.process_input(i) - if r['return']>0: return r + if r['return'] > 0: + return r # Take only out from original control - i['control']={'out':i['control']['out'], - 'common':True} + i['control'] = {'out': i['control']['out'], + 'common': True} tags_list = utils.convert_tags_to_list(i) - if 'automation' not in tags_list: tags_list.append('automation') + if 'automation' not in tags_list: + tags_list.append('automation') - i['meta']={'automation_alias':self.meta['alias'], - 'automation_uid':self.meta['uid'], - 'tags':tags_list} + i['meta'] = {'automation_alias': self.meta['alias'], + 'automation_uid': self.meta['uid'], + 'tags': tags_list} - if 'tags' in i: del(i['tags']) + if 'tags' in i: + del (i['tags']) # Use yaml by default if 'yaml' not in i: @@ -158,12 +166,13 @@ def add_cmx(self, i): # Pass to common action r_obj = self.cmind.x(i) - if r_obj['return']>0: return r_obj + if r_obj['return'] > 0: + return r_obj new_automation_path = r_obj['path'] if console: - print ('Created automation in {}'.format(new_automation_path)) + print('Created automation in {}'.format(new_automation_path)) module_name = 'modulex.py' @@ -176,18 +185,17 @@ def add_cmx(self, i): # Copy module files for f in ['modulex_dummy.py']: f1 = os.path.join(self.path, f) - f2 = os.path.join(new_automation_path, f.replace('_dummy','')) + f2 = os.path.join(new_automation_path, f.replace('_dummy', '')) if console: - print (' * Copying {} to {}'.format(f1, f2)) + print(' * Copying {} to {}'.format(f1, f2)) - shutil.copyfile(f1,f2) + shutil.copyfile(f1, f2) return r_obj - - ############################################################ + def doc(self, i): """ Add CM automation. diff --git a/cm/cmind/repo/automation/automation/module_misc.py b/cm/cmind/repo/automation/automation/module_misc.py index 985cc9d70..3ed1c7d1d 100644 --- a/cm/cmind/repo/automation/automation/module_misc.py +++ b/cm/cmind/repo/automation/automation/module_misc.py @@ -4,8 +4,10 @@ import os from cmind import utils - + ############################################################ + + def doc(i): """ Document CM automations. @@ -39,24 +41,25 @@ def doc(i): # Check parsed automation if 'parsed_automation' not in i: - return {'return':1, 'error':'automation is not specified'} + return {'return': 1, 'error': 'automation is not specified'} console = i.get('out') == 'con' - repos = i.get('repos','') - if repos == '': repos='internal,a4705959af8e447a' + repos = i.get('repos', '') + if repos == '': + repos = 'internal,a4705959af8e447a' - parsed_artifact = i.get('parsed_artifact',[]) - - if len(parsed_artifact)<1: - parsed_artifact = [('',''), ('','')] - elif len(parsed_artifact)<2: - parsed_artifact.append(('','')) + parsed_artifact = i.get('parsed_artifact', []) + + if len(parsed_artifact) < 1: + parsed_artifact = [('', ''), ('', '')] + elif len(parsed_artifact) < 2: + parsed_artifact.append(('', '')) else: repos = parsed_artifact[1][0] list_of_repos = repos.split(',') if ',' in repos else [repos] - + ii = utils.sub_input(i, self_module.cmind.cfg['artifact_keys']) ii['out'] = None @@ -64,63 +67,68 @@ def doc(i): # Search for automations in repos lst = [] for repo in list_of_repos: - parsed_artifact[1] = ('',repo) if utils.is_cm_uid(repo) else (repo,'') + parsed_artifact[1] = ('', repo) if utils.is_cm_uid( + repo) else (repo, '') ii['parsed_artifact'] = parsed_artifact r = self_module.search(ii) - if r['return']>0: return r + if r['return'] > 0: + return r lst += r['list'] toc = [] toc2 = {} md = [] - - for artifact in sorted(lst, key = lambda x: (-x.meta.get('sort',0), x.meta.get('alias',''))): - md_script_readme = ['*This README is automatically generated - don\'t edit! {{CM_SEE_README_EXTRA}} for extra notes!*', + + for artifact in sorted(lst, key=lambda x: (-x.meta.get('sort', 0), x.meta.get('alias', ''))): + md_script_readme = ['*This README is automatically generated - don\'t edit! {{CM_SEE_README_EXTRA}} for extra notes!*', '', ] - + path = artifact.path meta = artifact.meta original_meta = artifact.original_meta - print ('Documenting {}'.format(path)) + print('Documenting {}'.format(path)) - alias = meta.get('alias','') - uid = meta.get('uid','') + alias = meta.get('alias', '') + uid = meta.get('uid', '') - desc = meta.get('desc','') - developers = meta.get('developers','') + desc = meta.get('desc', '') + developers = meta.get('developers', '') repo_path = artifact.repo_path repo_meta = artifact.repo_meta - repo_alias = repo_meta.get('alias','') - repo_uid = repo_meta.get('uid','') + repo_alias = repo_meta.get('alias', '') + repo_uid = repo_meta.get('uid', '') # Check URL url = '' url_repo = '' if repo_alias == 'internal': url_repo = 'https://github.com/mlcommons/ck/tree/master/cm/cmind/repo' - url = url_repo+ '/automation/' + url = url_repo + '/automation/' elif '@' in repo_alias: - url_repo = 'https://github.com/'+repo_alias.replace('@','/')+'/tree/master' - if repo_meta.get('prefix','')!='': url_repo+='/'+repo_meta['prefix'] - url = url_repo+ '/automation/' + url_repo = 'https://github.com/' + \ + repo_alias.replace('@', '/')+'/tree/master' + if repo_meta.get('prefix', '') != '': + url_repo += '/'+repo_meta['prefix'] + url = url_repo + '/automation/' - if url!='': - url+=alias + if url != '': + url += alias x = '* [{}](#{})'.format(alias, alias) - if desc !='': x+=' *('+desc+')*' + if desc != '': + x += ' *('+desc+')*' toc.append(x) md.append('## '+alias) md.append('\n') - - if desc!='': + + if desc != '': md.append('*'+desc+'.*') md.append('\n') @@ -128,20 +136,22 @@ def doc(i): # md.append('Developers: '+developers) # md.append('\n') - md.append('* GitHub repository with CM automations: *cm pull [{}]({})*'.format(repo_alias, url_repo)) + md.append( + '* GitHub repository with CM automations: *cm pull [{}]({})*'.format(repo_alias, url_repo)) md.append('* CM automation code and meta: *[GitHub]({})*'.format(url)) md.append('* CM automation actions:') x = 'Automation actions' md_script_readme.append('### Automation actions') md_script_readme.append('') - + # Load module module_path = os.path.join(path, 'module.py') module_url = url+'/module.py' r = utils.load_txt(module_path) - if r['return']>0: return r + if r['return'] > 0: + return r module_py = r['string'] @@ -150,28 +160,36 @@ def doc(i): line_number = 0 for line in module_py_list: if line.startswith(' def '): - j=line.find('(') + j = line.find('(') action = line[8:j] if not action.startswith('_'): - x = ' * cm **'+action+'** '+alias+'    *( [See CM API]({}) )*'.format(module_url+'#L'+str(line_number)) + x = ' * cm **'+action+'** '+alias + \ + '    *( [See CM API]({}) )*'.format( + module_url+'#L'+str(line_number)) md.append(x) - y = '[add flags (dict keys) from this API]({})'.format(module_url+'#L'+str(line_number)) - y2 = '[add keys from this API]({})'.format(module_url+'#L'+str(line_number)) + y = '[add flags (dict keys) from this API]({})'.format( + module_url+'#L'+str(line_number)) + y2 = '[add keys from this API]({})'.format( + module_url+'#L'+str(line_number)) z = '({})'.format(y) - + md_script_readme.append('#### '+action) md_script_readme.append('') - md_script_readme.append(' * CM CLI: ```cm '+action+' '+alias+'``` '+z) - md_script_readme.append(' * CM CLI with UID: ```cm '+action+' '+alias+','+uid+'``` '+z) + md_script_readme.append( + ' * CM CLI: ```cm '+action+' '+alias+'``` '+z) + md_script_readme.append( + ' * CM CLI with UID: ```cm '+action+' '+alias+','+uid+'``` '+z) md_script_readme.append(' * CM Python API:') md_script_readme.append(' ```python') md_script_readme.append(' import cmind') md_script_readme.append('') md_script_readme.append(" r=cm.access({") - md_script_readme.append(" 'action':'{}'".format(action)) - md_script_readme.append(" 'automation':'{},{}'".format(alias,uid)) + md_script_readme.append( + " 'action':'{}'".format(action)) + md_script_readme.append( + " 'automation':'{},{}'".format(alias, uid)) md_script_readme.append(" 'out':'con'") md_script_readme.append(" ```") md_script_readme.append(" {}".format(y2)) @@ -182,36 +200,35 @@ def doc(i): md_script_readme.append(' ```') md_script_readme.append('') - - - line_number+=1 + line_number += 1 md.append('\n') # Add maintainers md_script_readme.append('### Maintainers') md_script_readme.append('') - md_script_readme.append('* [MLCommons taskforce on automation and reproducibility](https://github.com/mlcommons/ck/blob/master/docs/taskforce.md)') - + md_script_readme.append( + '* [MLCommons taskforce on automation and reproducibility](https://github.com/mlcommons/ck/blob/master/docs/taskforce.md)') # Check README and if it's already automatically generated path_readme = os.path.join(path, 'README.md') path_readme_extra = os.path.join(path, 'README-extra.md') if os.path.isfile(path_readme): - r = utils.load_txt(path_readme, split = True) - if r['return']>0: return - + r = utils.load_txt(path_readme, split=True) + if r['return'] > 0: + return + s = r['string'] readme = r['list'] if not 'automatically generated' in s.lower(): found_path_readme_extra = True - + # Attempt to rename to README-extra.md if os.path.isfile(path_readme_extra): - return {'return':1, 'error':'README.md is not auto-generated and README-extra.md already exists - can\'t rename'} + return {'return': 1, 'error': 'README.md is not auto-generated and README-extra.md already exists - can\'t rename'} os.rename(path_readme, path_readme_extra) @@ -219,7 +236,7 @@ def doc(i): os.chdir(path) os.system('git add README-extra.md') os.chdir(cur_dir) - + cm_readme_extra = '' cm_see_readme_extra = 'Use `README-extra.md`' @@ -227,25 +244,25 @@ def doc(i): readme_extra_url = url+'/README-extra.md' cm_see_readme_extra = 'See [extra README](README-extra.md)' - cm_readme_extra='\n'+cm_see_readme_extra+'.\n' - + cm_readme_extra = '\n'+cm_see_readme_extra+'.\n' s = '\n'.join(md_script_readme) s = s.replace('{{CM_SEE_README_EXTRA}}', cm_see_readme_extra) r = utils.save_txt(path_readme, s) - if r['return']>0: return r + if r['return'] > 0: + return r # Add to Git (if in git) os.chdir(path) os.system('git add README.md') os.chdir(cur_dir) - # Load template r = utils.load_txt(os.path.join(self_module.path, template_file)) - if r['return']>0: return r + if r['return'] > 0: + return r s = r['string'] @@ -253,13 +270,15 @@ def doc(i): s = s.replace('{{CM_MAIN}}', '\n'.join(md)) # Output - output_dir = i.get('output_dir','') + output_dir = i.get('output_dir', '') - if output_dir == '': output_dir = '..' + if output_dir == '': + output_dir = '..' output_file = os.path.join(output_dir, output_file) r = utils.save_txt(output_file, s) - if r['return']>0: return r + if r['return'] > 0: + return r - return {'return':0} + return {'return': 0} diff --git a/cm/cmind/repo/automation/ckx/module.py b/cm/cmind/repo/automation/ckx/module.py index 97cc73f2b..e1958da52 100644 --- a/cm/cmind/repo/automation/ckx/module.py +++ b/cm/cmind/repo/automation/ckx/module.py @@ -7,6 +7,7 @@ from cmind.automation import Automation from cmind import utils + class CAutomation(Automation): """ CM front-end to the legacy CK framework @@ -23,7 +24,7 @@ def any(self, i): Example: cm run ck program:*susan* - + Args: (CM input dict): pass to CK @@ -32,18 +33,18 @@ def any(self, i): * return (int): return code == 0 if no error and >0 if error * (error) (str): error string if return>0 - + * Output from the CK automation action - + """ import ck.kernel as ck - artifact = i.get('artifact','') + artifact = i.get('artifact', '') + + i['cid'] = artifact - i['cid']=artifact - if 'out' not in i: - i['out']='con' + i['out'] = 'con' return ck.access(i) diff --git a/cm/cmind/repo/automation/core/module.py b/cm/cmind/repo/automation/core/module.py index d0ac4ee9c..7aecf4100 100644 --- a/cm/cmind/repo/automation/core/module.py +++ b/cm/cmind/repo/automation/core/module.py @@ -9,51 +9,52 @@ # This is just an example of how to import extra files from such a package # We need to make it unique! -#import sys -#sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -#from cm_60cb625a46b38610 import misc +# import sys +# sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +# from cm_60cb625a46b38610 import misc + class CMInit(): ############################################################### - def run(self, quiet = False, skip = False, repo_name = 'mlcommons@cm4mlops', repo_url = '', - repo_branch = '', repo_checkout = '', x_was_called = False): + def run(self, quiet=False, skip=False, repo_name='mlcommons@cm4mlops', repo_url='', + repo_branch='', repo_checkout='', x_was_called=False): import cmind - print ('Checking platform information ...') + print('Checking platform information ...') self.get_sys_platform() - print ('[SUCCESS]') + print('[SUCCESS]') - print ('') - print ('Checking system dependencies ...') + print('') + print('Checking system dependencies ...') r = self.install_system_packages(quiet) - if r['return']>0 or r.get('warning','') !='' : + if r['return'] > 0 or r.get('warning', '') != '': return r - print ('[SUCCESS]') + print('[SUCCESS]') - rr = {'return':0} + rr = {'return': 0} # Do not pull extra repositories in CMX if not skip and not x_was_called: - print ('') - print ('Pulling default automation repository ...') + print('') + print('Pulling default automation repository ...') - print ('') - ii = {'action':'pull', - 'automation':'repo', - 'out':'con'} + print('') + ii = {'action': 'pull', + 'automation': 'repo', + 'out': 'con'} if repo_url != '': ii['url'] = repo_url elif repo_name != '': ii['artifact'] = repo_name - if repo_branch !='': + if repo_branch != '': ii['branch'] = repo_branch - if repo_checkout !='': + if repo_checkout != '': ii['checkout'] = repo_checkout rr = cmind.access(ii) @@ -68,12 +69,11 @@ def install_system_packages(self, quiet): # List of packages to install via system package manager packages = [] - + git_status = self.command_exists('git') if not git_status: packages.append("git") - # wget and curl are managed via CM scripts on Windows if os.name != 'nt': @@ -85,10 +85,10 @@ def install_system_packages(self, quiet): if not curl_status: packages.append("curl") - name='venv' + name = 'venv' if name in sys.modules: - pass #nothing needed + pass # nothing needed else: spec = importlib.util.find_spec(name) if spec is not None: @@ -97,7 +97,7 @@ def install_system_packages(self, quiet): spec.loader.exec_module(module) else: packages.append("python3-venv") - + warning = '' if packages: @@ -111,34 +111,37 @@ def install_system_packages(self, quiet): install_cmd = '{}apt-get update && apt-get install -y {}' if install_cmd == '': - warning = "You must install the following system packages manually: {}".format(', '.join(packages)) + warning = "You must install the following system packages manually: {}".format( + ', '.join(packages)) else: - print ('') - print ('The following system packages will be installed:') - print ('') - print (install_cmd.format('sudo', ' '.join(packages))) + print('') + print('The following system packages will be installed:') + print('') + print(install_cmd.format('sudo', ' '.join(packages))) sudo = 'sudo ' if not quiet: - print ('') - x = input ('Would you like to skip "sudo" from above command (y/N)? ') + print('') + x = input( + 'Would you like to skip "sudo" from above command (y/N)? ') - if x.lower() in ['y','yes']: + if x.lower() in ['y', 'yes']: sudo = '' install_cmd = install_cmd.format(sudo, ' '.join(packages)) - print ('') - print ('Running system command:') - print (install_cmd) + print('') + print('Running system command:') + print(install_cmd) r = os.system(install_cmd) - if r>0: - return {'return':1, 'error':f'Command {install_cmd} failed with return code {r}'} + if r > 0: + return {'return': 1, 'error': f'Command {install_cmd} failed with return code {r}'} - rr = {'return':0} + rr = {'return': 0} - if warning != '': rr['warning'] = warning + if warning != '': + rr['warning'] = warning return rr @@ -166,7 +169,8 @@ def get_package_manager_details(self): manager = self.detect_package_manager() if manager: try: - version_output = subprocess.check_output([manager, '--version'], stderr=subprocess.STDOUT).decode('utf-8') + version_output = subprocess.check_output( + [manager, '--version'], stderr=subprocess.STDOUT).decode('utf-8') return manager, version_output.split('\n')[0] except subprocess.CalledProcessError: return manager, 'Version information not available' @@ -177,7 +181,7 @@ def get_package_manager_details(self): # Checks if command exists(for installing required packages). # If the command exists, which returns 0, making the function return True. # If the command does not exist, which returns a non-zero value, making the function return False. - # NOTE: The standard output and standard error streams are redirected to PIPES so that it could be captured in future if needed. + # NOTE: The standard output and standard error streams are redirected to PIPES so that it could be captured in future if needed. def command_exists(self, command): import subprocess @@ -187,13 +191,12 @@ def command_exists(self, command): elif self.system == "Windows": return subprocess.call([command, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) == 0 - ############################################################### + def get_sys_platform(self): import platform - self.system = platform.system() - + self.system = platform.system() class CAutomation(Automation): @@ -205,8 +208,8 @@ class CAutomation(Automation): def __init__(self, cmind, automation_file): super().__init__(cmind, __file__) - ############################################################ + def uid(self, i): """ Generate CM UID. @@ -258,7 +261,7 @@ def init(self, i): repo_name = i.get('repo', '').strip() repo_url = i.get('url', '').strip() - if repo_url == '' and repo_name == '': + if repo_url == '' and repo_name == '': repo_name = 'mlcommons@cm4mlops' repo_branch = i.get('branch', '') @@ -269,18 +272,19 @@ def init(self, i): if hasattr(self.cmind, 'x_was_called'): x_was_called = self.cmind.x_was_called - r = cm_init.run(quiet = quiet, - skip = skip, - repo_name = repo_name, - repo_url = repo_url, - repo_branch = repo_branch, - repo_checkout = repo_checkout, - x_was_called = x_was_called) - if r['return']>0: return r + r = cm_init.run(quiet=quiet, + skip=skip, + repo_name=repo_name, + repo_url=repo_url, + repo_branch=repo_branch, + repo_checkout=repo_checkout, + x_was_called=x_was_called) + if r['return'] > 0: + return r warning = r.get('warning', '') if warning != '': - print ('') - print (warning) + print('') + print(warning) - return {'return':0} + return {'return': 0} diff --git a/cm/cmind/repo/automation/core/module_misc.py b/cm/cmind/repo/automation/core/module_misc.py index 0fadae23b..4bcb549cc 100644 --- a/cm/cmind/repo/automation/core/module_misc.py +++ b/cm/cmind/repo/automation/core/module_misc.py @@ -5,6 +5,8 @@ import cmind.utils as utils ############################################################ + + def uid(i): console = i.get('out') == 'con' @@ -12,6 +14,6 @@ def uid(i): r = utils.gen_uid() if console: - print (r['uid']) + print(r['uid']) return r diff --git a/cm/cmind/repo/automation/repo/module.py b/cm/cmind/repo/automation/repo/module.py index 2b7578eb1..3af99ac81 100644 --- a/cm/cmind/repo/automation/repo/module.py +++ b/cm/cmind/repo/automation/repo/module.py @@ -8,6 +8,7 @@ from cmind import utils from cmind import net + class CAutomation(Automation): """ CM "repo" automation actions @@ -80,39 +81,39 @@ def pull(self, i): if '@' not in alias: alias = self.cmind.cfg['repo_url_org'] + '@' + alias - url += alias.replace('@','/') + url += alias.replace('@', '/') if pat != '' and url.startswith('https://'): url = url[:8]+pat+'@'+url[8:] else: - if alias == '': - # Get alias from URL - alias = url - - # Check if zip file - j = alias.find('.zip') - if j>0: - j1 = alias.rfind('/') - alias = alias[j1+1:j+4] - else: - if alias.endswith('.git'): alias=alias[:-4] - - if alias.startswith('git@'): - j = alias.find(':') - if j>=0: - alias = alias[j+1:].replace('/','@') - else: - j = alias.find('//') - if j>=0: - j1 = alias.find('/', j+2) - if j1>=0: - alias = alias[j1+1:].replace('/','@') + if alias == '': + # Get alias from URL + alias = url + + # Check if zip file + j = alias.find('.zip') + if j > 0: + j1 = alias.rfind('/') + alias = alias[j1+1:j+4] + else: + if alias.endswith('.git'): + alias = alias[:-4] + if alias.startswith('git@'): + j = alias.find(':') + if j >= 0: + alias = alias[j+1:].replace('/', '@') + else: + j = alias.find('//') + if j >= 0: + j1 = alias.find('/', j+2) + if j1 >= 0: + alias = alias[j1+1:].replace('/', '@') if url == '': pull_repos = [] - for repo in sorted(self.cmind.repos.lst, key = lambda x: x.meta.get('alias','')): + for repo in sorted(self.cmind.repos.lst, key=lambda x: x.meta.get('alias', '')): meta = repo.meta if meta.get('git', False): @@ -124,41 +125,33 @@ def pull(self, i): pull_repos.append({'alias': os.path.basename(repo_path), 'path_to_repo': repo_path}) else: - # We are migrating cm-mlops repo from mlcommons@ck to a clean and new mlcommons@cm4mlops: - # https://github.com/mlcommons/ck/issues/1215 - # As discussed, we should have a transparent redirect with a warning - # unless branch/checkout is used - in such case we keep old repository - # for backwards compatibility and reproducibility - + # Migration has been completed branch = i.get('branch', '') new_branch = i.get('new_branch', '') checkout = i.get('checkout', '') _dir = i.get('dir', '') - r = net.request({'get': {'action': 'check-migration-repo-notes', 'repo': url, 'branch': branch, 'checkout': checkout}}) - notes = r.get('dict', {}).get('notes','') - if notes !='': - print (notes) - if alias == 'mlcommons@ck' and branch == '' and checkout == '' and _dir == '': - print ('=========================================================================') - print ('Warning: mlcommons@ck was automatically changed to mlcommons@cm4mlops.') - print ('If you want to use older mlcommons@ck repository, use branch or checkout.') - print ('=========================================================================') + print( + '=========================================================================') + print( + 'Warning: mlcommons@ck was automatically changed to mlcommons@cm4mlops.') + print( + 'If you want to use older mlcommons@ck repository, use branch or checkout.') + print( + '=========================================================================') alias = 'mlcommons@cm4mlops' url = url.replace('mlcommons/ck', 'mlcommons/cm4mlops') - - pull_repos = [{'alias':alias, - 'url':url, + pull_repos = [{'alias': alias, + 'url': url, 'branch': branch, 'new_branch': new_branch, 'checkout': checkout, 'dir': _dir, 'depth': i.get('depth', '')}] - # Go through repositories and pull repo_meta = {} repo_metas = {} @@ -167,75 +160,75 @@ def pull(self, i): # if not self.cmind.xlogger == None: # self.cmind.log(f"x repo log: {pull_repos}", "debug") - + for repo in pull_repos: - alias = repo['alias'] - url = repo.get('url', '') - branch = repo.get('branch','') - new_branch = repo.get('new_branch','') - checkout = repo.get('checkout','') - depth = repo.get('depth','') - path_to_repo = repo.get('path_to_repo', None) - _dir = repo.get('dir', '') - - if console: - print (self.cmind.cfg['line']) - print ('Alias: {}'.format(alias)) - if url!='': - print ('URL: {}'.format(url)) - if branch!='': - print ('Branch: {}'.format(branch)) - if new_branch!='': - print ('New branch: {}'.format(new_branch)) - if checkout!='': - print ('Checkout: {}'.format(checkout)) - if _dir!='': - print ('Directory: {}'.format(_dir)) - if depth!='' and depth!=None: - print ('Depth: {}'.format(str(depth))) - print ('') - - # Prepare path to repo - repos = self.cmind.repos - - r = repos.pull(alias = alias, - url = url, - branch = branch, - new_branch = new_branch, - checkout = checkout, - _dir = _dir, - console = console, - desc=desc, - prefix=prefix, - depth=depth, - path_to_repo=path_to_repo, - checkout_only=checkout_only, - skip_zip_parent_dir=skip_zip_parent_dir, - extra_cmd_git = extra_cmd_git, - extra_cmd_pip = extra_cmd_pip) - if r['return']>0: return r - - repo_meta = r['meta'] - - repo_metas[alias] = repo_meta - - if len(r.get('warnings', []))>0: - warnings += r['warnings'] - - if len(pull_repos)>0 and self.cmind.use_index: + alias = repo['alias'] + url = repo.get('url', '') + branch = repo.get('branch', '') + new_branch = repo.get('new_branch', '') + checkout = repo.get('checkout', '') + depth = repo.get('depth', '') + path_to_repo = repo.get('path_to_repo', None) + _dir = repo.get('dir', '') + if console: - print (self.cmind.cfg['line']) + print(self.cmind.cfg['line']) + print('Alias: {}'.format(alias)) + if url != '': + print('URL: {}'.format(url)) + if branch != '': + print('Branch: {}'.format(branch)) + if new_branch != '': + print('New branch: {}'.format(new_branch)) + if checkout != '': + print('Checkout: {}'.format(checkout)) + if _dir != '': + print('Directory: {}'.format(_dir)) + if depth != '' and depth != None: + print('Depth: {}'.format(str(depth))) + print('') + + # Prepare path to repo + repos = self.cmind.repos + + r = repos.pull(alias=alias, + url=url, + branch=branch, + new_branch=new_branch, + checkout=checkout, + _dir=_dir, + console=console, + desc=desc, + prefix=prefix, + depth=depth, + path_to_repo=path_to_repo, + checkout_only=checkout_only, + skip_zip_parent_dir=skip_zip_parent_dir, + extra_cmd_git=extra_cmd_git, + extra_cmd_pip=extra_cmd_pip) + if r['return'] > 0: + return r - ii = {'out':'con'} if console else {} - rx = self.reindex(ii) + repo_meta = r['meta'] - print_warnings(warnings) + repo_metas[alias] = repo_meta - return {'return':0, 'meta':repo_meta, 'metas': repo_metas} + if len(r.get('warnings', [])) > 0: + warnings += r['warnings'] + if len(pull_repos) > 0 and self.cmind.use_index: + if console: + print(self.cmind.cfg['line']) + ii = {'out': 'con'} if console else {} + rx = self.reindex(ii) + + print_warnings(warnings) + + return {'return': 0, 'meta': repo_meta, 'metas': repo_metas} ############################################################ + def checkout(self, i): """ Checkout repository @@ -254,8 +247,8 @@ def checkout(self, i): return self.pull(i) - ############################################################ + def show(self, i): """ Show verbose info about registered CM repos. @@ -271,8 +264,8 @@ def show(self, i): return self.search(i) - ############################################################ + def search(self, i): """ List registered CM repos. @@ -305,12 +298,14 @@ def search(self, i): lst = [] - parsed_artifact = i.get('parsed_artifact',[]) + parsed_artifact = i.get('parsed_artifact', []) - min_out = i.get('min',False) + min_out = i.get('min', False) - artifact_obj = parsed_artifact[0] if len(parsed_artifact)>0 else ('','') - artifact_repo = parsed_artifact[1] if len(parsed_artifact)>1 else None + artifact_obj = parsed_artifact[0] if len( + parsed_artifact) > 0 else ('', '') + artifact_repo = parsed_artifact[1] if len( + parsed_artifact) > 1 else None artifact_repo_wildcards = False if artifact_repo is not None: @@ -321,13 +316,14 @@ def search(self, i): meta = repo.meta uid = meta['uid'] - alias = meta.get('alias','') + alias = meta.get('alias', '') - r = utils.match_objects(uid = uid, - alias = alias, - uid2 = artifact_obj[1], - alias2 = artifact_obj[0]) - if r['return']>0: return r + r = utils.match_objects(uid=uid, + alias=alias, + uid2=artifact_obj[1], + alias2=artifact_obj[0]) + if r['return'] > 0: + return r if r['match']: lst.append(repo) @@ -343,78 +339,81 @@ def search(self, i): path = l.path_with_prefix if with_prefix else l.path if min_out: - print (path) + print(path) else: if i.get('verbose', False): - uid = meta['uid'] - desc = meta.get('desc', '') - git = meta.get('git', False) - prefix = meta.get('prefix', '') - - local_alias = os.path.basename(path) - - if alias == 'local' or alias == 'internal' or local_alias == 'ck': - local_alias = '' - - print (self.cmind.cfg['line']) - print ('Path: {}'.format(path)) - if prefix != '': - print (' CM sub-directory: {}'.format(prefix)) - - if alias != local_alias: - if local_alias!='': - print (' Alias: {}'.format(local_alias)) - print (' Original alias: {}'.format(alias)) - else: - print (' Alias: {}'.format(alias)) - else: - print (' Alias: {}'.format(alias)) - - print (' UID: {}'.format(uid)) - - if desc != '': - print ('Description: {}'.format(desc)) - - print ('Git: {}'.format(str(git))) - - if git: - url = '' - branch = '' - checkout = '' - - import subprocess - - cur_dir = os.getcwd() - - os.chdir(path) - - try: - url = subprocess.check_output('git config --get remote.origin.url', shell=True).decode("utf-8").strip() - except subprocess.CalledProcessError as e: - url = '' - - try: - branch = subprocess.check_output('git rev-parse --abbrev-ref HEAD', shell=True).decode("utf-8").strip() - except subprocess.CalledProcessError as e: - branch = '' - - try: - checkout = subprocess.check_output('git rev-parse HEAD', shell=True).decode("utf-8").strip() - except subprocess.CalledProcessError as e: - checkout = '' - - if url!='': - print (f' URL: {url}') - if branch!='': - print (f' Branch: {branch}') - if checkout!='': - print (f' Checkout: {checkout}') + uid = meta['uid'] + desc = meta.get('desc', '') + git = meta.get('git', False) + prefix = meta.get('prefix', '') + + local_alias = os.path.basename(path) + + if alias == 'local' or alias == 'internal' or local_alias == 'ck': + local_alias = '' + + print(self.cmind.cfg['line']) + print('Path: {}'.format(path)) + if prefix != '': + print(' CM sub-directory: {}'.format(prefix)) + + if alias != local_alias: + if local_alias != '': + print(' Alias: {}'.format(local_alias)) + print(' Original alias: {}'.format(alias)) + else: + print(' Alias: {}'.format(alias)) + else: + print(' Alias: {}'.format(alias)) + + print(' UID: {}'.format(uid)) + + if desc != '': + print('Description: {}'.format(desc)) + + print('Git: {}'.format(str(git))) + + if git: + url = '' + branch = '' + checkout = '' + + import subprocess + + cur_dir = os.getcwd() + + os.chdir(path) + + try: + url = subprocess.check_output( + 'git config --get remote.origin.url', shell=True).decode("utf-8").strip() + except subprocess.CalledProcessError as e: + url = '' + + try: + branch = subprocess.check_output( + 'git rev-parse --abbrev-ref HEAD', shell=True).decode("utf-8").strip() + except subprocess.CalledProcessError as e: + branch = '' + + try: + checkout = subprocess.check_output( + 'git rev-parse HEAD', shell=True).decode("utf-8").strip() + except subprocess.CalledProcessError as e: + checkout = '' + + if url != '': + print(f' URL: {url}') + if branch != '': + print(f' Branch: {branch}') + if checkout != '': + print(f' Checkout: {checkout}') else: - print ('{},{} = {}'.format(alias, uid, path)) + print('{},{} = {}'.format(alias, uid, path)) - return {'return':0, 'list':lst} + return {'return': 0, 'list': lst} ############################################################ def where(self, i): @@ -435,7 +434,7 @@ def where(self, i): * list (list): list of updated CM repository objects """ - i['min']=True + i['min'] = True return self.search(i) @@ -463,18 +462,19 @@ def update(self, i): for repo in self.cmind.repos.lst: meta = repo.meta - alias = meta.get('alias','') + alias = meta.get('alias', '') git = meta.get('git', False) if git: if console: - print ('Updating "{}" ...'.format(alias)) - print ('') - r = self.cmind.repos.pull(alias = alias, console = console) - if r['return']>0: return r + print('Updating "{}" ...'.format(alias)) + print('') + r = self.cmind.repos.pull(alias=alias, console=console) + if r['return'] > 0: + return r - return {'return':0, 'list':self.cmind.repos.lst} + return {'return': 0, 'list': self.cmind.repos.lst} ############################################################ def delete(self, i): @@ -505,39 +505,41 @@ def delete(self, i): console = i.get('out') == 'con' - force = (i.get('force',False) or i.get('f',False)) + force = (i.get('force', False) or i.get('f', False)) - artifact = i.get('artifact','').strip() + artifact = i.get('artifact', '').strip() if artifact == '': - return {'return':1, 'error':'repositories are not specified'} + return {'return': 1, 'error': 'repositories are not specified'} # Search CM repository - i['action']='search' - i['out']=None - r=self.cmind.access(i) + i['action'] = 'search' + i['out'] = None + r = self.cmind.access(i) - if r['return'] >0 : return r + if r['return'] > 0: + return r lst = r['list'] - if len(lst)==0: - return {'return':1, 'error':'Repository not found'} + if len(lst) == 0: + return {'return': 1, 'error': 'Repository not found'} remove_all = i.get('all', '') - r = self.cmind.repos.delete(lst, remove_all = remove_all, console = console, force = force) + r = self.cmind.repos.delete( + lst, remove_all=remove_all, console=console, force=force) if self.cmind.use_index: - ii = {'out':'con'} if console else {} + ii = {'out': 'con'} if console else {} rx = self.reindex(ii) - + return r ############################################################ def ximport(self, i): - if i.get('path','')!='': - i['here']=True + if i.get('path', '') != '': + i['here'] = True return self.init(i) @@ -570,57 +572,59 @@ def init(self, i): console = i.get('out') == 'con' alias = i.get('artifact', '') - uid = i.get('uid','') + uid = i.get('uid', '') path = i.get('path', '') - desc = i.get('desc','') - prefix = i.get('prefix','') + desc = i.get('desc', '') + prefix = i.get('prefix', '') # If path is not specified, initialize in the current directory! - if (path=='' and alias=='') or (alias!='' and (path=='.' or i.get('here', False))): - path = os.path.abspath(os.getcwd()) + if (path == '' and alias == '') or (alias != '' and (path == '.' or i.get('here', False))): + path = os.path.abspath(os.getcwd()) # Check if there is a repo in a path - r = self.cmind.access({'action':'detect', - 'automation':self.meta['alias']+','+self.meta['uid'], - 'path':path}) - if r['return']>0: return r + r = self.cmind.access({'action': 'detect', + 'automation': self.meta['alias']+','+self.meta['uid'], + 'path': path}) + if r['return'] > 0: + return r repo_desc_found = r['found'] - repo_registered = r.get('registered',False) + repo_registered = r.get('registered', False) if repo_desc_found: if repo_registered: - return {'return':1, 'error':'CM repository found in this path and already registered in CM'} + return {'return': 1, 'error': 'CM repository found in this path and already registered in CM'} if console: - print ('CM repository description found in this path: {}'.format(r['path_to_repo_desc'])) + print('CM repository description found in this path: {}'.format( + r['path_to_repo_desc'])) - path= r['path_to_repo'] + path = r['path_to_repo'] repo_meta = r['meta'] - repo_meta_alias = repo_meta.get('alias','') - repo_meta_uid = repo_meta.get('uid','') - repo_meta_prefix = repo_meta.get('prefix','') + repo_meta_alias = repo_meta.get('alias', '') + repo_meta_uid = repo_meta.get('uid', '') + repo_meta_prefix = repo_meta.get('prefix', '') # Check if mismatch in alias - if alias!='': - if repo_meta_alias!='' and repo_meta_alias!=alias: - return {'return':1, 'error':'mismatch between new repo alias and the existing one'} + if alias != '': + if repo_meta_alias != '' and repo_meta_alias != alias: + return {'return': 1, 'error': 'mismatch between new repo alias and the existing one'} else: alias = repo_meta_alias # Check if mismatch in UID - if uid!='': - if repo_meta_uid!='' and repo_meta_uid!=uid: - return {'return':1, 'error':'mismatch between new repo UID and the existing one'} + if uid != '': + if repo_meta_uid != '' and repo_meta_uid != uid: + return {'return': 1, 'error': 'mismatch between new repo UID and the existing one'} else: uid = repo_meta_uid # Check if mismatch in prefix - if prefix!='': - if repo_meta_prefix!='' and repo_meta_prefix is not None and repo_meta_prefix!=prefix: - return {'return':1, 'error':'mismatch between new repo prefix and the existing one'} + if prefix != '': + if repo_meta_prefix != '' and repo_meta_prefix is not None and repo_meta_prefix != prefix: + return {'return': 1, 'error': 'mismatch between new repo prefix and the existing one'} else: prefix = repo_meta_prefix @@ -630,43 +634,48 @@ def init(self, i): alias = os.path.basename(path).strip().lower() # Generate UID - if uid =='': - r=utils.gen_uid() - if r['return']>0: return r + if uid == '': + r = utils.gen_uid() + if r['return'] > 0: + return r uid = r['uid'] - if alias == '': alias = uid + if alias == '': + alias = uid if console: - print (self.cmind.cfg['line']) - print ('Alias: {}'.format(alias)) - print ('UID: {}'.format(uid)) - print ('Desc: {}'.format(desc)) - print ('Prefix: {}'.format(prefix)) - print ('') + print(self.cmind.cfg['line']) + print('Alias: {}'.format(alias)) + print('UID: {}'.format(uid)) + print('Desc: {}'.format(desc)) + print('Prefix: {}'.format(prefix)) + print('') # Check if repository with this alias and UID doesn't exist for repo_artifact in [alias, uid]: - ii={'automation':'repo', 'action':'search', 'artifact':repo_artifact} + ii = {'automation': 'repo', 'action': 'search', + 'artifact': repo_artifact} - r=self.cmind.access(ii) - if r['return']>0: return r + r = self.cmind.access(ii) + if r['return'] > 0: + return r - lst=r['list'] + lst = r['list'] - if len(lst)>0: - return {'return':1, 'error':'Repository "{}" is already registered in CM'.format(repo_artifact)} + if len(lst) > 0: + return {'return': 1, 'error': 'Repository "{}" is already registered in CM'.format(repo_artifact)} - # Create repository - r = self.cmind.repos.init(alias = alias, uid = uid, path = path, console = console, desc=desc, prefix=prefix, only_register=repo_desc_found) + # Create repository + r = self.cmind.repos.init(alias=alias, uid=uid, path=path, console=console, + desc=desc, prefix=prefix, only_register=repo_desc_found) if self.cmind.use_index: - ii = {'out':'con'} if console else {} + ii = {'out': 'con'} if console else {} rx = self.reindex(ii) - + warnings = r.get('warnings', []) print_warnings(warnings) - + return r ############################################################ @@ -704,17 +713,18 @@ def pack(self, i): console = i.get('out') == 'con' # Search repository - i['out']=None + i['out'] = None r = self.search(i) - if r['return']>0: return r + if r['return'] > 0: + return r lst = r['list'] - if len(lst)==0: - return {'return':16, 'error':'no repsitories found'} - elif len(lst)>1: - return {'return':16, 'error':'more than 1 repository found'} + if len(lst) == 0: + return {'return': 16, 'error': 'no repsitories found'} + elif len(lst) > 1: + return {'return': 16, 'error': 'more than 1 repository found'} repo = lst[0] @@ -723,10 +733,12 @@ def pack(self, i): pack_file = self.cmind.cfg['default_repo_pack'] if console: - print ('Packing repo from {} to {} ...'.format(repo_path, pack_file)) + print('Packing repo from {} to {} ...'.format(repo_path, pack_file)) - r = utils.list_all_files({'path': repo_path, 'all': 'yes', 'ignore_names':['.git']}) - if r['return'] > 0: return r + r = utils.list_all_files( + {'path': repo_path, 'all': 'yes', 'ignore_names': ['.git']}) + if r['return'] > 0: + return r files = r['list'] @@ -745,7 +757,7 @@ def pack(self, i): except Exception as e: return {'return': 1, 'error': 'failed to pack repo: {}'.format(format(e))} - return {'return':0} + return {'return': 0} ############################################################ def unpack(self, i): @@ -773,15 +785,15 @@ def unpack(self, i): pack_file = self.cmind.cfg['default_repo_pack'] if not os.path.isfile(pack_file): - return {'return':16, 'error':'CM repo pack file not found {}'.format(pack_file)} + return {'return': 16, 'error': 'CM repo pack file not found {}'.format(pack_file)} - # Attempt to read cmr.json + # Attempt to read cmr.json repo_pack_file = open(pack_file, 'rb') repo_pack_zip = zipfile.ZipFile(repo_pack_file) repo_pack_desc = self.cmind.cfg['file_meta_repo'] - files=repo_pack_zip.namelist() + files = repo_pack_zip.namelist() repo_meta = {} @@ -804,28 +816,30 @@ def unpack(self, i): # Get meta info uid = repo_meta['uid'] alias = repo_meta['alias'] - desc = repo_meta.get('desc','') - prefix = repo_meta.get('prefix','') + desc = repo_meta.get('desc', '') + prefix = repo_meta.get('prefix', '') # Check if repo exists - r = self.search({'parsed_artifact':[(alias,uid)]}) - if r['return']>0: return r + r = self.search({'parsed_artifact': [(alias, uid)]}) + if r['return'] > 0: + return r lst = r['list'] - if len(lst)>0: - return {'return':1, 'error':'Repository already exists'} + if len(lst) > 0: + return {'return': 1, 'error': 'Repository already exists'} # Initialize repository - r_new = self.init({'artifact':alias, - 'uid':uid, - 'prefix':prefix}) - if r_new['return']>0: return r_new + r_new = self.init({'artifact': alias, + 'uid': uid, + 'prefix': prefix}) + if r_new['return'] > 0: + return r_new repo_path = r_new['path_to_repo'] if console: - print ('Unpacking {} to {} ...'.format(pack_file, repo_path)) + print('Unpacking {} to {} ...'.format(pack_file, repo_path)) # Unpacking zip for f in files: @@ -849,9 +863,9 @@ def unpack(self, i): repo_pack_file.close() if self.cmind.use_index: - ii = {'out':'con'} if console else {} + ii = {'out': 'con'} if console else {} rx = self.reindex(ii) - + return r_new ############################################################################## @@ -874,36 +888,39 @@ def import_ck_to_cm(self, i): import cmind import os - r=ck.access({'action':'search', - 'module_uoa':'repo', - 'add_meta':'yes'}) - if r['return']>0: return r + r = ck.access({'action': 'search', + 'module_uoa': 'repo', + 'add_meta': 'yes'}) + if r['return'] > 0: + return r - lst=r['lst'] + lst = r['lst'] for l in lst: - ruoa=l['data_uoa'] - ruid=l['data_uid'] + ruoa = l['data_uoa'] + ruid = l['data_uid'] - rmeta=l['meta'] + rmeta = l['meta'] - rpath=rmeta.get('path','') + rpath = rmeta.get('path', '') - if rpath!='' and os.path.isdir(rpath): - print ('********************************************************') - print (ruoa) - print (rpath) + if rpath != '' and os.path.isdir(rpath): + print('********************************************************') + print(ruoa) + print(rpath) - r=cmind.access({'automation':'repo', - 'action':'init', - 'artifact':ruoa, - 'path':rpath}) - if r['return']>0: return r + r = cmind.access({'automation': 'repo', + 'action': 'init', + 'artifact': ruoa, + 'path': rpath}) + if r['return'] > 0: + return r r = convert_ck_dir_to_cm(rpath) - if r['return']>0: return r + if r['return'] > 0: + return r - return {'return':0} + return {'return': 0} ############################################################################## def convert_ck_to_cm(self, i): @@ -927,19 +944,19 @@ def convert_ck_to_cm(self, i): import cmind import os - console = i.get('out') == 'con' # Search CM repository - i['action']='search' - i['out']=None - r=self.cmind.access(i) + i['action'] = 'search' + i['out'] = None + r = self.cmind.access(i) - if r['return'] >0 : return r + if r['return'] > 0: + return r lst = r['list'] - if len(lst)==0: - return {'return':1, 'error':'Repository not found'} + if len(lst) == 0: + return {'return': 1, 'error': 'Repository not found'} for repo in lst: @@ -947,19 +964,20 @@ def convert_ck_to_cm(self, i): ruid = repo.meta['uid'] rpath = repo.path - if rpath!='' and os.path.isdir(rpath): - print ('********************************************************') - print (ralias) - print (ruid) - print (rpath) + if rpath != '' and os.path.isdir(rpath): + print('********************************************************') + print(ralias) + print(ruid) + print(rpath) r = convert_ck_dir_to_cm(rpath) - if r['return']>0: return r - - return {'return':0} + if r['return'] > 0: + return r + return {'return': 0} ############################################################ + def detect(self, i): """ Detect CM repository in the current path. @@ -1010,15 +1028,17 @@ def detect(self, i): repos = self.cmind.repos path = i.get('path') - if path == '': path = None + if path == '': + path = None # Search for cmr.yaml or cmr.json in current directory and above - r=utils.find_file_in_current_directory_or_above([self.cmind.cfg['file_meta_repo']+'.json', - self.cmind.cfg['file_meta_repo']+'.yaml'], - path_to_start = path) - if r['return']>0: return r + r = utils.find_file_in_current_directory_or_above([self.cmind.cfg['file_meta_repo']+'.json', + self.cmind.cfg['file_meta_repo']+'.yaml'], + path_to_start=path) + if r['return'] > 0: + return r - rr = {'return':0} + rr = {'return': 0} found = r['found'] rr['found'] = found @@ -1036,7 +1056,8 @@ def detect(self, i): # Load meta r = utils.load_json_or_yaml(path_to_repo_desc) - if r['return']>0: return r + if r['return'] > 0: + return r repo_meta = r['meta'] @@ -1054,7 +1075,8 @@ def detect(self, i): registered = True if registered: - rr['cm_repo'] = utils.assemble_cm_object(repo_meta['alias'],repo_meta['uid']) + rr['cm_repo'] = utils.assemble_cm_object( + repo_meta['alias'], repo_meta['uid']) rr['registered'] = registered @@ -1063,15 +1085,17 @@ def detect(self, i): if found: for reverse in [False, True]: - r=utils.find_file_in_current_directory_or_above([self.cmind.cfg['file_cmeta']+'.json', - self.cmind.cfg['file_cmeta']+'.yaml'], - path_to_start = path, - reverse = reverse, - path_to_stop = path_to_repo_real) - if r['return']>0: return r + r = utils.find_file_in_current_directory_or_above([self.cmind.cfg['file_cmeta']+'.json', + self.cmind.cfg['file_cmeta']+'.yaml'], + path_to_start=path, + reverse=reverse, + path_to_stop=path_to_repo_real) + if r['return'] > 0: + return r artifact_found = r['found'] - artifact_found_in_current_path = r.get('found_in_current_path', False) + artifact_found_in_current_path = r.get( + 'found_in_current_path', False) rr['artifact_found'] = artifact_found @@ -1088,62 +1112,69 @@ def detect(self, i): # Load meta path_to_artifact_desc_without_ext = path_to_artifact_desc j = path_to_artifact_desc.rfind('.') - if j>=0: - path_to_artifact_desc_without_ext=path_to_artifact_desc[:j] + if j >= 0: + path_to_artifact_desc_without_ext = path_to_artifact_desc[:j] - r = utils.load_yaml_and_json(path_to_artifact_desc_without_ext) - if r['return']>0: return r + r = utils.load_yaml_and_json( + path_to_artifact_desc_without_ext) + if r['return'] > 0: + return r artifact_meta = r['meta'] rr['artifact_meta'] = artifact_meta - rr['cm_automation'] = utils.assemble_cm_object(artifact_meta['automation_alias'],artifact_meta['automation_uid']) + rr['cm_automation'] = utils.assemble_cm_object( + artifact_meta['automation_alias'], artifact_meta['automation_uid']) if not reverse or artifact_found_in_current_path: - rr['cm_artifact'] = utils.assemble_cm_object(artifact_meta['alias'],artifact_meta['uid']) + rr['cm_artifact'] = utils.assemble_cm_object( + artifact_meta['alias'], artifact_meta['uid']) break # Print if console: if not found: - print ('CM repository not found') + print('CM repository not found') else: - print ('CM repository found:') + print('CM repository found:') - print ('') - print ('Path to repo desc file: {}'.format(path_to_repo_desc)) - print ('Path to repo: {}'.format(path_to_repo)) + print('') + print('Path to repo desc file: {}'.format(path_to_repo_desc)) + print('Path to repo: {}'.format(path_to_repo)) if registered: - print ('') - print (' This repository is registered in CM') + print('') + print(' This repository is registered in CM') - print ('') - print (' Repo alias: {}'.format(repo_meta['alias'])) - print (' Repo UID: {}'.format(repo_meta['uid'])) + print('') + print(' Repo alias: {}'.format(repo_meta['alias'])) + print(' Repo UID: {}'.format(repo_meta['uid'])) if artifact_found: - print ('') - print ('Current directory has related CM automation:') + print('') + print('Current directory has related CM automation:') - print ('') - print (' Automation alias: {}'.format(artifact_meta['automation_alias'])) - print (' Automation UID: {}'.format(artifact_meta['automation_uid'])) + print('') + print(' Automation alias: {}'.format( + artifact_meta['automation_alias'])) + print(' Automation UID: {}'.format( + artifact_meta['automation_uid'])) if artifact_found_in_current_path: - print ('') - print ('Current directory has a CM artifact:') + print('') + print('Current directory has a CM artifact:') - print ('') - print (' Artifact alias: {}'.format(artifact_meta['alias'])) - print (' Artifact UID: {}'.format(artifact_meta['uid'])) + print('') + print(' Artifact alias: {}'.format( + artifact_meta['alias'])) + print(' Artifact UID: {}'.format(artifact_meta['uid'])) return rr - ############################################################ + def reindex(self, i): """ Reindex all CM repositories @@ -1166,28 +1197,29 @@ def reindex(self, i): console = i.get('out') == 'con' if console: - print ('') - print ('Reindexing all CM artifacts. Can take some time ...') - + print('') + print('Reindexing all CM artifacts. Can take some time ...') + out = 'con' if verbose else '' - + # Clean index self.cmind.index.meta = {} - + import time t1 = time.time() - r = self.cmind.access({'action':'search', - 'automation':'*', - 'out':out, - 'skip_index_search':True, - 'force_index_add':True}) - if r['return']>0: return r + r = self.cmind.access({'action': 'search', + 'automation': '*', + 'out': out, + 'skip_index_search': True, + 'force_index_add': True}) + if r['return'] > 0: + return r t2 = time.time() - t1 if console: - print ('Took {:.1f} sec.'.format(t2)) + print('Took {:.1f} sec.'.format(t2)) # Save rx = self.cmind.index.save() @@ -1197,9 +1229,7 @@ def reindex(self, i): rx = self.cmind.index.load() # Ignore output for now to continue working even if issues ... - - return {'return':0, 'self_time':t2} - + return {'return': 0, 'self_time': t2} ############################################################################## @@ -1224,30 +1254,31 @@ def convert_ck_dir_to_cm(rpath): dir1 = os.listdir(rpath) for d1 in dir1: - if d1!='.cm': - dd1=os.path.join(rpath, d1) + if d1 != '.cm': + dd1 = os.path.join(rpath, d1) if os.path.isdir(dd1): dir2 = os.listdir(dd1) for d2 in dir2: - if d2!='.cm': - dd2=os.path.join(dd1, d2, '.cm') + if d2 != '.cm': + dd2 = os.path.join(dd1, d2, '.cm') if os.path.isdir(dd2): dmeta = {} broken = False - for f in ['info.json','meta.json']: - dd2a=os.path.join(dd2, f) + for f in ['info.json', 'meta.json']: + dd2a = os.path.join(dd2, f) if os.path.isfile(dd2a): - r=ck.load_json_file({'json_file':dd2a}) - if r['return']>0: return r + r = ck.load_json_file({'json_file': dd2a}) + if r['return'] > 0: + return r dmeta.update(r['dict']) else: - broken=True + broken = True break if broken or \ @@ -1255,42 +1286,46 @@ def convert_ck_dir_to_cm(rpath): 'backup_module_uid' not in dmeta or \ 'backup_module_uoa' not in dmeta: - print ('Ignored: '+dd2) + print('Ignored: '+dd2) else: - backup_data_uid = dmeta['backup_data_uid'] + backup_data_uid = dmeta['backup_data_uid'] + + backup_module_uid = dmeta['backup_module_uid'] + backup_module_uoa = dmeta['backup_module_uoa'] - backup_module_uid = dmeta['backup_module_uid'] - backup_module_uoa = dmeta['backup_module_uoa'] + dmeta2 = {} + for k in dmeta: + if not k.startswith('backup_') and not k.startswith('cm_'): + dmeta2[k] = dmeta[k] - dmeta2 = {} - for k in dmeta: - if not k.startswith('backup_') and not k.startswith('cm_'): - dmeta2[k]=dmeta[k] + dmeta2['uid'] = backup_data_uid - dmeta2['uid']=backup_data_uid + if not ck.is_uid(d2): + dmeta2['alias'] = d2 - if not ck.is_uid(d2): - dmeta2['alias']=d2 + dmeta2['automation_alias'] = backup_module_uoa + dmeta2['automation_uid'] = backup_module_uid - dmeta2['automation_alias']=backup_module_uoa - dmeta2['automation_uid']=backup_module_uid + cm_meta_file = os.path.join( + dd1, d2, '_cm.json') - cm_meta_file = os.path.join(dd1, d2, '_cm.json') + print(cm_meta_file) - print (cm_meta_file) + r = ck.save_json_to_file( + {'json_file': cm_meta_file, 'dict': dmeta2, 'sort_keys': 'yes'}) + if r['return'] > 0: + return r - r=ck.save_json_to_file({'json_file':cm_meta_file, 'dict': dmeta2, 'sort_keys':'yes'}) - if r['return']>0: return r + return {'return': 0} - return {'return':0} def print_warnings(warnings): - if len(warnings)>0: - print ('') - print ('WARNINGS:') - print ('') + if len(warnings) > 0: + print('') + print('WARNINGS:') + print('') for w in warnings: - print (' {}'.format(w)) - + print(' {}'.format(w)) + return From 617408fa5b39e65d7a81309dad213b148fd571b4 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Mon, 9 Dec 2024 10:41:02 +0100 Subject: [PATCH 2/3] fixing test version --- cm/cmind/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cm/cmind/__init__.py b/cm/cmind/__init__.py index a8087a912..0ca06d95d 100644 --- a/cm/cmind/__init__.py +++ b/cm/cmind/__init__.py @@ -2,7 +2,7 @@ # # Written by Grigori Fursin -__version__ = "3.5.1" +__version__ = "3.5.1.1" from cmind.core import access from cmind.core import x From 78c34ed6c98781b15180b2d45b24c3022879efa3 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Mon, 9 Dec 2024 22:18:15 +0100 Subject: [PATCH 3/3] added utils.get_memory_use --- cm/CHANGES.md | 1 + cm/cmind/utils.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/cm/CHANGES.md b/cm/CHANGES.md index 698ef7493..460124635 100644 --- a/cm/CHANGES.md +++ b/cm/CHANGES.md @@ -1,4 +1,5 @@ ## V3.5.1.1 + - added utils.get_memory_use - formatted Python modules from the internal repository using autopep8 ## V3.5.1 diff --git a/cm/cmind/utils.py b/cm/cmind/utils.py index 5ba5bbce1..7801fe0be 100644 --- a/cm/cmind/utils.py +++ b/cm/cmind/utils.py @@ -2170,3 +2170,52 @@ def substitute_template(template, variables): except KeyError as e: return f"Error: Missing value for {e.args[0]} in the vars dictionary." +############################################################################## +def get_memory_use(console = False): + + """ + Get memory usage + + Args: + console (bool): if True, print to console + + Returns: + memory_use (int) + memory_use_gb (float) + available_memory (int) + available_memory_gb (float) + total_memory (int) + total_memory_gb (float) + + """ + + import os + import psutil + + pid = os.getpid() + + python_process = psutil.Process(pid) + + memory_use = python_process.memory_info()[0] # in bytes + memory_use_gb = memory_use / (1024 ** 3) + + memory_info = psutil.virtual_memory() + + available_memory = memory_info.available # in bytes + total_memory = memory_info.total # in bytes + + available_memory_gb = available_memory / (1024 ** 3) + total_memory_gb = total_memory / (1024 ** 3) + + if console: + print(f"Total Memory: {total_memory_gb:.2f} GB") + print(f"Available Memory: {available_memory_gb:.2f} GB") + print(f"Used Python Memory: {memory_use_gb:.2f} GB") + + return {'return':0, 'memory_use': memory_use, + 'memory_use_gb': memory_use_gb, + 'available_memory': available_memory, + 'available_memory_gb': available_memory_gb, + 'total_memory': total_memory, + 'total_memory_gb': total_memory_gb} +