Use json logging to log information about the issue being processed when exceptions occur

Label_Microservice/deployment/overlays/dev/kustomization.yaml

@@ -2,6 +2,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 bases:
 - ../../base
+images:
+- digest: sha256:cb2b2e604d4056b78ecd51d7113de04ebfa60e542310265b3871e7873417e34a
+  name: gcr.io/issue-label-bot-dev/bot-worker
+  newName: gcr.io/issue-label-bot-dev/bot-worker:3a82547
 commonLabels:
   environment: dev
 namespace: label-bot-dev

Label_Microservice/deployment/overlays/prod/kustomization.yaml

@@ -8,6 +8,6 @@ patchesStrategicMerge:
 resources:
 - ../../base
 images:
-- digest: sha256:cb2b2e604d4056b78ecd51d7113de04ebfa60e542310265b3871e7873417e34a
-  name: gcr.io/issue-label-bot-dev/bot-worker
-  newName: gcr.io/issue-label-bot-dev/bot-worker:3a82547
+- name: gcr.io/issue-label-bot-dev/bot-worker
+  newName: gcr.io/issue-label-bot-dev/bot-worker
+  newTag: 79cd85a-dirty

py/label_microservice/worker.py

@@ -20,6 +20,8 @@
 
 DEFAULT_APP_URL = "https://github.com/marketplace/issue-label-bot"
 
+DEFAULT_APP_URL = "https://github.com/marketplace/issue-label-bot"
+
 class Worker:
     """
     The worker class aims to do label prediction for issues from github repos.
@@ -147,28 +149,46 @@ def callback(message):
             repo_name = message.attributes['repo_name']
             issue_num = message.attributes['issue_num']
 
+            # log the prediction, which will be used to track the performance
+            # TODO(https://github.com/kubeflow/code-intelligence/issues/79)
+            # Ensure we capture the information needed to measure performance
+            # in stackdriver
+            log_dict = {
+                'repo_owner': repo_owner,
+                'repo_name': repo_name,
+                'issue_num': int(issue_num),
+            }
+
             data = {
                 "repo_owner": repo_owner,
                 "repo_name": repo_name,
                 "issue_num": issue_num,
             }
             try:
                 predictions = self._predictor.predict(data)
+                log_dict['predictions'] = predictions
                 self.add_labels_to_issue(installation_id, repo_owner, repo_name,
                                          issue_num, predictions)
 
-                # log the prediction, which will be used to track the performance
-                # TODO(https://github.com/kubeflow/code-intelligence/issues/79)
-                # Ensure we capture the information needed to measure performance
-                # in stackdriver
-                log_dict = {
-                    'repo_owner': repo_owner,
-                    'repo_name': repo_name,
-                    'issue_num': int(issue_num),
-                    'predictions': predictions,
-                }
                 logging.info("Add labels to issue.", extra=log_dict)
 
+            # TODO(jlewi): I observed cases where some of the initial inferences
+            # would succeed but on subsequent ones it started failing
+            # see: https://github.com/kubeflow/code-intelligence/issues/70#issuecomment-570491289
+            # Restarting is a bit of a hack. We should try to figure out
+            # why its happening and fix it.
+            except tf_errors.FailedPreconditionError as e:
+                logging.fatal(f"Exception occurred while handling issue "
+                              f"{repo_owner}/{repo_name}#{issue_num}. \n"
+                              f"Exception: {e}\n"
+                              f"{traceback.format_exc()}\n."
+                              f"This usually indicates an issue with "
+                              f"trying to use the model in a thread different "
+                              f"from the one it was created in. "
+                              f"The program will restart to try to recover.",
+                              extra=log_dict)
+                sys.exit(1)
+
             # TODO(jlewi): I observed cases where some of the initial inferences
             # would succeed but on subsequent ones it started failing
             # see: https://github.com/kubeflow/code-intelligence/issues/70#issuecomment-570491289
@@ -196,7 +216,7 @@ def callback(message):
                 logging.error(f"Exception occurred while handling issue "
                               f"{repo_owner}/{repo_name}#{issue_num}. \n"
                               f"Exception: {e}\n"
-                              f"{traceback.format_exc()}")
+                              f"{traceback.format_exc()}", extra=log_dict)
 
             # acknowledge the message, or pubsub will repeatedly attempt to deliver it
             message.ack()