From 66c213728d7e1c77d34a4e661ef6e24cdb772b7c Mon Sep 17 00:00:00 2001 From: Joshua Garde Date: Thu, 7 Mar 2024 15:41:03 -0800 Subject: [PATCH 1/2] Update pkg command to allow importing of containers without url and importing of user rules --- sdscli/adapters/hysds/pkg.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/sdscli/adapters/hysds/pkg.py b/sdscli/adapters/hysds/pkg.py index 78ac7b2..e459e11 100644 --- a/sdscli/adapters/hysds/pkg.py +++ b/sdscli/adapters/hysds/pkg.py @@ -4,6 +4,7 @@ from __future__ import division from __future__ import absolute_import from builtins import open +from datetime import datetime from future import standard_library standard_library.install_aliases() @@ -23,6 +24,8 @@ JOB_SPECS_INDEX = "job_specs" HYSDS_IOS_MOZART_INDEX = "hysds_ios-mozart" HYSDS_IOS_GRQ_INDEX = "hysds_ios-grq" +USER_RULES_MOZART_INDEX = 'user_rules-mozart' +USER_RULES_GRQ_INDEX = 'user_rules-grq' mozart_es = get_mozart_es() @@ -62,9 +65,10 @@ def export(args): validate_dir(export_dir) # create export directory - # download container - get(cont_info['url'], export_dir) - cont_info['url'] = os.path.basename(cont_info['url']) + # download container if url provided + if cont_info.get('url', None) != None: + get(cont_info['url'], export_dir) + cont_info['url'] = os.path.basename(cont_info['url']) query = { "query": { @@ -218,11 +222,13 @@ def import_pkg(args): logger.debug("code_bucket: %s" % code_bucket) logger.debug("code_bucket_url: %s" % code_bucket_url) - # upload container image to s3 cont_info = manifest['containers'] - cont_image = os.path.join(export_dir, cont_info['url']) - cont_info['url'] = "{}/{}".format(code_bucket_url, cont_info['url']) - put(cont_image, cont_info['url']) + + # upload container image to s3 + if cont_info.get('url', None) != None: + cont_image = os.path.join(export_dir, cont_info['url']) + cont_info['url'] = "{}/{}".format(code_bucket_url, cont_info['url']) + put(cont_image, cont_info['url']) # index container in ES indexed_container = mozart_es.index_document(index=CONTAINERS_INDEX, body=cont_info, id=cont_info['id']) @@ -260,6 +266,19 @@ def import_pkg(args): indexed_hysds_io = mozart_es.index_document(index=HYSDS_IOS_GRQ_INDEX, body=hysds_io, id=hysds_io_id) logger.debug(indexed_hysds_io) + # index user_rules to ES + for component in (('mozart', USER_RULES_MOZART_INDEX), ('grq', USER_RULES_GRQ_INDEX)): + for rule in manifest.get('user_rules', {}).get(component[0], []): + now = datetime.utcnow().isoformat() + 'Z' + + if not rule.get('creation_time', None): + rule['creation_time'] = now + if not rule.get('modified_time', None): + rule['modified_time'] = now + + result = mozart_es.index_document(index=component[1], body=rule) # indexing user rules + logger.debug(result) + shutil.rmtree(export_dir) # remove package dir From 26643cb30a3071d576a0110edefb23de036738e7 Mon Sep 17 00:00:00 2001 From: Joshua Garde Date: Thu, 7 Mar 2024 17:15:49 -0800 Subject: [PATCH 2/2] Version bump -> 1.3.2 --- sdscli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdscli/__init__.py b/sdscli/__init__.py index 65594f2..7fe6999 100644 --- a/sdscli/__init__.py +++ b/sdscli/__init__.py @@ -3,6 +3,6 @@ from __future__ import division from __future__ import absolute_import -__version__ = "1.3.1" +__version__ = "1.3.2" __url__ = "https://github.com/sdskit/sdscli" __description__ = "Command line interface for SDSKit"