From df5e8327eaa8b94293e250869a6d037744347959 Mon Sep 17 00:00:00 2001 From: John-Paul Navarro Date: Tue, 23 May 2023 07:17:46 -0500 Subject: [PATCH] See CHANGELOG tag v2.3.0 --- CHANGELOG | 4 ++++ bin/router_cider.py | 25 +++++++++++++++---------- sbin/router_cider.sh | 2 +- sbin/router_cider.sysconfig | 2 +- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index ae46fd2..2f876b8 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,7 @@ +v2.3.0 20230523 JP + - Remove hardcoded logging level + - Improved debug logging + v2.2.0 20230187 JP - Process new online_service_resources diff --git a/bin/router_cider.py b/bin/router_cider.py index 70c130f..c37c571 100755 --- a/bin/router_cider.py +++ b/bin/router_cider.py @@ -166,17 +166,16 @@ def Setup(self): # The affiliations we are processing self.AFFILIATIONS = set(self.config.get('AFFILIATIONS', ['ACCESS', 'XSEDE'])) - # The affiliations for a resource compiled at load time - self.resource_AFFILIATIONS = {} if self.args.daemonaction == 'start': if self.src['scheme'] not in ['http', 'https'] or self.dest['scheme'] not in ['warehouse']: self.logger.error('Can only daemonize when source=[http|https] and destination=warehouse') sys.exit(1) - self.logger.info('Source: ' + self.src['display']) - self.logger.info('Destination: ' + self.dest['display']) - self.logger.info('Config: ' + self.config_file) + self.logger.info('Source: {}'.format(self.src['display'])) + self.logger.info('Destination: {}'.format(self.dest['display'])) + self.logger.info('Config: {}' .format(self.config_file)) + self.logger.info('Log Level: {}({})'.format(loglevel_str, loglevel_num)) self.logger.info('Affiliations: ' + ', '.join(self.AFFILIATIONS)) def SaveDaemonStdOut(self, path): @@ -204,22 +203,27 @@ def exit(self, rc): sys.exit(rc) def Retrieve_Infrastructure(self, url): + # The affiliations for a resource compiled at load time + self.resource_AFFILIATIONS = {} + infra_all = [] for AFF in self.AFFILIATIONS: infra_aff = self.Retrieve_Affiliation_Infrastructure(url, affiliation=AFF) if not infra_aff: continue if 'resources' not in infra_aff: - self.logger.error('CiDeR JSON response (affiliation={}) is missing a \'resources\' element'.format(AFF)) + self.logger.error('CiDeR JSON response for affiliation={} is missing a \'resources\' element'.format(AFF)) continue + self.logger.debug('Retrieved affiliation={} {}/items'.format(AFF, len(infra_aff['resources']))) for item in infra_aff['resources']: - # Collapses multiple occurances of a rsource into one resource and accumulate its affiliations for later + # Collapses multiple occurances of a resource into one resource and accumulate its affiliations for later id = item['resource_id'] if id in self.resource_AFFILIATIONS: self.resource_AFFILIATIONS[id].add(AFF) continue self.resource_AFFILIATIONS[id] = set([AFF]) - infra_all.append(item) + infra_all.append(item) # We only need each item once even if it has multiple affiliations + self.logger.debug('Retrieved from all affiliations {}/items'.format(len(infra_all))) return(infra_all) def Retrieve_Affiliation_Infrastructure(self, url, affiliation='XSEDE'): @@ -345,6 +349,7 @@ def Warehouse_Info(self, info_json): self.new = {} # New resources in document for item in CiderInfrastructure.objects.all(): self.cur[item.cider_resource_id] = item + self.logger.debug('Retrieved from database {}/items'.format(len(self.cur))) for p_res in info_json: # Iterating over parent resources # Require affiliation, a resource_id, and an information services ResourceID @@ -394,7 +399,7 @@ def Warehouse_Info(self, info_json): 'updated_at': p_res['updated_at'] }) model.save() - self.logger.debug('Base ID={}, ResourceID={}'.format(id, infoid)) + self.logger.debug('Base ID={}, ResourceID={}, created={}'.format(id, infoid, created)) self.new[id]=model self.stats['Update'] += 1 except (DataError, IntegrityError) as e: @@ -433,7 +438,7 @@ def Warehouse_Info(self, info_json): 'updated_at': s_res['updated_at'] }) model.save() - self.logger.debug(' Sub ID={}, ResourceID={}, Type={}'.format(s_res[id_lookup[subtype]], s_res['parent_resource']['info_resourceid'], type_lookup[subtype])) + self.logger.debug(' Sub ID={}, ResourceID={}, Type={}, created={}'.format(s_res[id_lookup[subtype]], s_res['parent_resource']['info_resourceid'], type_lookup[subtype], created)) self.new[s_res[id_lookup[subtype]]]=model self.stats['Update'] += 1 except (DataError, IntegrityError) as e: diff --git a/sbin/router_cider.sh b/sbin/router_cider.sh index 4e44fb8..460c44b 100755 --- a/sbin/router_cider.sh +++ b/sbin/router_cider.sh @@ -30,7 +30,7 @@ if [[ "$1" != --pdb && "$2" != --pdb && "$3" != --pdb && "$4" != --pdb ]]; then fi APP_BIN=${APP_SOURCE}/bin/${APP_NAME}.py -APP_OPTS="-l info -c ${APP_HOME}/conf/${APP_NAME}.conf" +APP_OPTS="-c ${APP_HOME}/conf/${APP_NAME}.conf" PYTHON_BIN=python3 export LD_LIBRARY_PATH=${PYTHON_BASE}/lib diff --git a/sbin/router_cider.sysconfig b/sbin/router_cider.sysconfig index c43f944..c8703fb 100644 --- a/sbin/router_cider.sysconfig +++ b/sbin/router_cider.sysconfig @@ -2,7 +2,7 @@ APP_NAME=%APP_NAME% APP_HOME=%APP_HOME% APP_BIN=%APP_HOME%/PROD/bin/%APP_NAME%.py # Add any command line options for your daemon here -APP_OPTS="-l info -c %APP_HOME%/conf/%APP_NAME%.conf" +APP_OPTS="-c %APP_HOME%/conf/%APP_NAME%.conf" PYTHON_BASE=%PYTHON_BASE% PIPENV_BASE=%APP_HOME%/python