Fix logging in the new worker images.

* Attach extra metadata information in the log messages. * Related to #70 - use ensemble models
kubeflow · Jan 18, 2020 · 2ccd3e7 · 2ccd3e7
1 parent a7fe608
commit 2ccd3e7
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 4 deletions.
diff --git a/Label_Microservice/deployment/overlays/dev/kustomization.yaml b/Label_Microservice/deployment/overlays/dev/kustomization.yaml
@@ -2,6 +2,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 bases:
 - ../../base
+images:
+- digest: sha256:cb2b2e604d4056b78ecd51d7113de04ebfa60e542310265b3871e7873417e34a
+  name: gcr.io/issue-label-bot-dev/bot-worker
+  newName: gcr.io/issue-label-bot-dev/bot-worker:3a82547
 commonLabels:
   environment: dev
 namespace: label-bot-dev

diff --git a/Label_Microservice/deployment/overlays/prod/kustomization.yaml b/Label_Microservice/deployment/overlays/prod/kustomization.yaml
@@ -8,6 +8,6 @@ patchesStrategicMerge:
 resources:
 - ../../base
 images:
-- digest: sha256:cb2b2e604d4056b78ecd51d7113de04ebfa60e542310265b3871e7873417e34a
-  name: gcr.io/issue-label-bot-dev/bot-worker
-  newName: gcr.io/issue-label-bot-dev/bot-worker:3a82547
+- name: gcr.io/issue-label-bot-dev/bot-worker
+  newName: gcr.io/issue-label-bot-dev/bot-worker
+  newTag: 57c6d26
diff --git a/py/label_microservice/worker.py b/py/label_microservice/worker.py
@@ -20,6 +20,8 @@
 
 DEFAULT_APP_URL = "https://github.com/marketplace/issue-label-bot"
 
+DEFAULT_APP_URL = "https://github.com/marketplace/issue-label-bot"
+
 class Worker:
     """
     The worker class aims to do label prediction for issues from github repos.
@@ -169,6 +171,23 @@ def callback(message):
                 }
                 logging.info("Add labels to issue.", extra=log_dict)
 
+            # TODO(jlewi): I observed cases where some of the initial inferences
+            # would succeed but on subsequent ones it started failing
+            # see: https://github.com/kubeflow/code-intelligence/issues/70#issuecomment-570491289
+            # Restarting is a bit of a hack. We should try to figure out
+            # why its happening and fix it.
+            except tf_errors.FailedPreconditionError as e:
+                logging.fatal(f"Exception occurred while handling issue "
+                              f"{repo_owner}/{repo_name}#{issue_num}. \n"
+                              f"Exception: {e}\n"
+                              f"{traceback.format_exc()}\n."
+                              f"This usually indicates an issue with "
+                              f"trying to use the model in a thread different "
+                              f"from the one it was created in. "
+                              f"The program will restart to try to recover.",
+                              extra=log_dict)
+                sys.exit(1)
+
             # TODO(jlewi): I observed cases where some of the initial inferences
             # would succeed but on subsequent ones it started failing
             # see: https://github.com/kubeflow/code-intelligence/issues/70#issuecomment-570491289
@@ -196,7 +215,7 @@ def callback(message):
                 logging.error(f"Exception occurred while handling issue "
                               f"{repo_owner}/{repo_name}#{issue_num}. \n"
                               f"Exception: {e}\n"
-                              f"{traceback.format_exc()}")
+                              f"{traceback.format_exc()}", extra=log_dict)
 
             # acknowledge the message, or pubsub will repeatedly attempt to deliver it
             message.ack()