See CHANGELOG tag v2.3.0

CHANGELOG

@@ -1,3 +1,7 @@
+v2.3.0 20230523 JP
+ - Remove hardcoded logging level
+ - Improved debug logging
+
 v2.2.0 20230187 JP
  - Process new online_service_resources
 

bin/router_cider.py

@@ -166,17 +166,16 @@ def Setup(self):
 
  # The affiliations we are processing
  self.AFFILIATIONS = set(self.config.get('AFFILIATIONS', ['ACCESS', 'XSEDE']))
- # The affiliations for a resource compiled at load time
- self.resource_AFFILIATIONS = {}
 
  if self.args.daemonaction == 'start':
  if self.src['scheme'] not in ['http', 'https'] or self.dest['scheme'] not in ['warehouse']:
  self.logger.error('Can only daemonize when source=[http|https] and destination=warehouse')
  sys.exit(1)
 
- self.logger.info('Source: ' + self.src['display'])
- self.logger.info('Destination: ' + self.dest['display'])
- self.logger.info('Config: ' + self.config_file)
+ self.logger.info('Source: {}'.format(self.src['display']))
+ self.logger.info('Destination: {}'.format(self.dest['display']))
+ self.logger.info('Config: {}' .format(self.config_file))
+ self.logger.info('Log Level: {}({})'.format(loglevel_str, loglevel_num))
  self.logger.info('Affiliations: ' + ', '.join(self.AFFILIATIONS))
 
  def SaveDaemonStdOut(self, path):
@@ -204,22 +203,27 @@ def exit(self, rc):
  sys.exit(rc)
 
  def Retrieve_Infrastructure(self, url):
+ # The affiliations for a resource compiled at load time
+ self.resource_AFFILIATIONS = {}
+
  infra_all = []
  for AFF in self.AFFILIATIONS:
  infra_aff = self.Retrieve_Affiliation_Infrastructure(url, affiliation=AFF)
  if not infra_aff:
  continue
  if 'resources' not in infra_aff:
- self.logger.error('CiDeR JSON response (affiliation={}) is missing a \'resources\' element'.format(AFF))
+ self.logger.error('CiDeR JSON response for affiliation={} is missing a \'resources\' element'.format(AFF))
  continue
+ self.logger.debug('Retrieved affiliation={} {}/items'.format(AFF, len(infra_aff['resources'])))
  for item in infra_aff['resources']:
- # Collapses multiple occurances of a rsource into one resource and accumulate its affiliations for later
+ # Collapses multiple occurances of a resource into one resource and accumulate its affiliations for later
  id = item['resource_id']
  if id in self.resource_AFFILIATIONS:
  self.resource_AFFILIATIONS[id].add(AFF)
  continue
  self.resource_AFFILIATIONS[id] = set([AFF])
- infra_all.append(item)
+ infra_all.append(item) # We only need each item once even if it has multiple affiliations
+ self.logger.debug('Retrieved from all affiliations {}/items'.format(len(infra_all)))
  return(infra_all)
 
  def Retrieve_Affiliation_Infrastructure(self, url, affiliation='XSEDE'):
@@ -345,6 +349,7 @@ def Warehouse_Info(self, info_json):
  self.new = {} # New resources in document
  for item in CiderInfrastructure.objects.all():
  self.cur[item.cider_resource_id] = item
+ self.logger.debug('Retrieved from database {}/items'.format(len(self.cur)))
 
  for p_res in info_json: # Iterating over parent resources
  # Require affiliation, a resource_id, and an information services ResourceID
@@ -394,7 +399,7 @@ def Warehouse_Info(self, info_json):
  'updated_at': p_res['updated_at']
  })
  model.save()
- self.logger.debug('Base ID={}, ResourceID={}'.format(id, infoid))
+ self.logger.debug('Base ID={}, ResourceID={}, created={}'.format(id, infoid, created))
  self.new[id]=model
  self.stats['Update'] += 1
  except (DataError, IntegrityError) as e:
@@ -433,7 +438,7 @@ def Warehouse_Info(self, info_json):
  'updated_at': s_res['updated_at']
  })
  model.save()
- self.logger.debug(' Sub ID={}, ResourceID={}, Type={}'.format(s_res[id_lookup[subtype]], s_res['parent_resource']['info_resourceid'], type_lookup[subtype]))
+ self.logger.debug(' Sub ID={}, ResourceID={}, Type={}, created={}'.format(s_res[id_lookup[subtype]], s_res['parent_resource']['info_resourceid'], type_lookup[subtype], created))
  self.new[s_res[id_lookup[subtype]]]=model
  self.stats['Update'] += 1
  except (DataError, IntegrityError) as e:

sbin/router_cider.sh

@@ -30,7 +30,7 @@ if [[ "$1" != --pdb && "$2" != --pdb && "$3" != --pdb && "$4" != --pdb ]]; then
 fi
 
 APP_BIN=${APP_SOURCE}/bin/${APP_NAME}.py
-APP_OPTS="-l info -c ${APP_HOME}/conf/${APP_NAME}.conf"
+APP_OPTS="-c ${APP_HOME}/conf/${APP_NAME}.conf"
 
 PYTHON_BIN=python3
 export LD_LIBRARY_PATH=${PYTHON_BASE}/lib

sbin/router_cider.sysconfig

@@ -2,7 +2,7 @@ APP_NAME=%APP_NAME%
 APP_HOME=%APP_HOME%
 APP_BIN=%APP_HOME%/PROD/bin/%APP_NAME%.py
 # Add any command line options for your daemon here
-APP_OPTS="-l info -c %APP_HOME%/conf/%APP_NAME%.conf"
+APP_OPTS="-c %APP_HOME%/conf/%APP_NAME%.conf"
 
 PYTHON_BASE=%PYTHON_BASE%
 PIPENV_BASE=%APP_HOME%/python