Add metadata to json decode errors

augur/tasks/git/dependency_tasks/core.py

@@ -1,12 +1,13 @@
 from datetime import datetime
 import os
 from augur.application.db.models import *
 from augur.application.db.lib import bulk_insert_dicts, get_repo_by_repo_git, get_value, get_session
 from augur.tasks.github.util.github_api_key_handler import GithubApiKeyHandler
 from augur.tasks.git.dependency_tasks.dependency_util import dependency_calculator as dep_calc
 from augur.tasks.util.worker_util import parse_json_from_subprocess_call
 from augur.tasks.git.util.facade_worker.facade_worker.utilitymethods import get_absolute_repo_path
 from augur.tasks.github.util.github_random_key_auth import GithubRandomKeyAuth
+from augur.tasks.util.metadata_exception import MetadataException
 
 
 def generate_deps_data(logger, repo_git):
@@ -49,7 +50,7 @@
 
         logger.info(f"Inserted {len(deps)} dependencies for repo {repo_id}")
 
 """
 def deps_model(session, repo_id,repo_git,repo_path,repo_name):
     # Data collection and storage method
 
@@ -82,7 +83,7 @@
 
     #setting the environmental variable which is required by scorecard
 
     with get_session() as session:
         #key_handler = GithubRandomKeyAuth(logger)
         key_handler = GithubApiKeyHandler(logger)       
         os.environ['GITHUB_AUTH_TOKEN'] = key_handler.get_random_key()
@@ -94,50 +95,47 @@
 
     try: 
         required_output = parse_json_from_subprocess_call(logger,['./scorecard', command, '--format=json'],cwd=path_to_scorecard)
-    except Exception as e: 
-        logger.error(f"Could not parse required output! Error: {e}")
-        raise e        
-
-    # end
 
-    logger.info('adding to database...')
-    logger.debug(f"output: {required_output}")
+        logger.info('adding to database...')
+        logger.debug(f"output: {required_output}")
 
-    if not required_output['checks']:
-        logger.info('No scorecard checks found!')
-        return
-
-    #Store the overall score first
-    to_insert = []
-    overall_deps_scorecard = {
-        'repo_id': repo_id,
-        'name': 'OSSF_SCORECARD_AGGREGATE_SCORE',
-        'scorecard_check_details': required_output['repo'],
-        'score': required_output['score'],
-        'tool_source': 'scorecard_model',
-        'tool_version': '0.43.9',
-        'data_source': 'Git',
-        'data_collection_date': datetime.now().strftime('%Y-%m-%dT%H:%M:%SZ')
-    }
-    to_insert.append(overall_deps_scorecard)
-   # bulk_insert_dicts(overall_deps_scorecard, RepoDepsScorecard, ["repo_id","name"])
-
-    #Store misc data from scorecard in json field. 
-    for check in required_output['checks']:
-        repo_deps_scorecard = {
+        if not required_output['checks']:
+            logger.info('No scorecard checks found!')
+            return
+
+        #Store the overall score first
+        to_insert = []
+        overall_deps_scorecard = {
             'repo_id': repo_id,
-            'name': check['name'],
-            'scorecard_check_details': check,
-            'score': check['score'],
+            'name': 'OSSF_SCORECARD_AGGREGATE_SCORE',
+            'scorecard_check_details': required_output['repo'],
+            'score': required_output['score'],
             'tool_source': 'scorecard_model',
             'tool_version': '0.43.9',
             'data_source': 'Git',
             'data_collection_date': datetime.now().strftime('%Y-%m-%dT%H:%M:%SZ')
         }
-        to_insert.append(repo_deps_scorecard)
-
-    bulk_insert_dicts(logger, to_insert, RepoDepsScorecard, ["repo_id","name"])
-
-    logger.info(f"Done generating scorecard for repo {repo_id} from path {path}")
+        to_insert.append(overall_deps_scorecard)
+    # bulk_insert_dicts(overall_deps_scorecard, RepoDepsScorecard, ["repo_id","name"])
 
+        #Store misc data from scorecard in json field. 
+        for check in required_output['checks']:
+            repo_deps_scorecard = {
+                'repo_id': repo_id,
+                'name': check['name'],
+                'scorecard_check_details': check,
+                'score': check['score'],
+                'tool_source': 'scorecard_model',
+                'tool_version': '0.43.9',
+                'data_source': 'Git',
+                'data_collection_date': datetime.now().strftime('%Y-%m-%dT%H:%M:%SZ')
+            }
+            to_insert.append(repo_deps_scorecard)
+
+        bulk_insert_dicts(logger, to_insert, RepoDepsScorecard, ["repo_id","name"])
+
+        logger.info(f"Done generating scorecard for repo {repo_id} from path {path}")
 
+    except Exception as e: 
+
+        raise MetadataException(e, f"required_output: {required_output}")

augur/tasks/util/metadata_exception.py

@@ -0,0 +1,6 @@
+class MetadataException(Exception):
+    def __init__(self, original_exception, additional_metadata):
+        self.original_exception = original_exception
+        self.additional_metadata = additional_metadata
+
+        super().__init__(f"{str(self.original_exception)} | Additional metadata: {self.additional_metadata}")

augur/tasks/util/worker_util.py

@@ -1,4 +1,4 @@
 #SPDX-License-Identifier: MIT
 import json
 #import gunicorn.app.base
 import numpy as np
@@ -11,6 +11,9 @@
 import json
 import subprocess
 
+from augur.tasks.util.metadata_exception import MetadataException
+
+
 def create_grouped_task_load(*args,processes=8,dataList=[],task=None):
 
     if not dataList or not task:
@@ -70,7 +73,7 @@
         # if a KeyError does not occur then a dict with those values has already been processed
         # if a KeyError occurs a dict with those values has not been found yet
         try:
             unique_values[key]
             continue
         except KeyError:
             unique_values[key] = 1
@@ -108,7 +111,7 @@
 
 def calculate_date_weight_from_timestamps(added,last_collection,domain_start_days=30):
     #Get the time since last collection as well as when the repo was added.
     if last_collection is None:
         delta = datetime.now() - added
         return date_weight_factor(delta.days)
     else:
@@ -117,7 +120,7 @@
         factor = date_weight_factor(delta.days,domain_shift=domain_start_days)
 
         #If the repo is older than thirty days, start to decrease its weight.
         if delta.days >= domain_start_days:
             return factor
         else:
             #Else increase its weight
@@ -126,9 +129,9 @@
 def parse_json_from_subprocess_call(logger, subprocess_arr, cwd=None):
     logger.info(f"running subprocess {subprocess_arr[0]}")
     if cwd:
         p = subprocess.run(subprocess_arr,cwd=cwd,capture_output=True, text=True, timeout=None)
     else:
         p = subprocess.run(subprocess_arr,capture_output=True, text=True, timeout=None)
 
     logger.info('subprocess completed... ')
 
@@ -141,7 +144,7 @@
             required_output = {}
     except json.decoder.JSONDecodeError as e:
         logger.error(f"Could not parse required output! \n output: {output} \n Error: {e}")
-        raise e
+        raise MetadataException(e, f"output : {output}")
 
     return required_output