[Aishwarya|Jenson] integrate with sentry

gojek · Feb 12, 2019 · 8d7ae20 · 8d7ae20
1 parent 0b3a773
commit 8d7ae20
Show file tree

Hide file tree

Showing 17 changed files with 137 additions and 22 deletions.
diff --git a/proctord/.env.sample b/proctord/.env.sample
@@ -27,4 +27,4 @@ export PROCTOR_MAIL_PASSWORD="password"
 export PROCTOR_MAIL_SERVER_HOST="smtp.mail.com"
 export PROCTOR_MAIL_SERVER_PORT="123"
 export PROCTOR_JOB_POD_ANNOTATIONS="{\"key.one\":\"true\"}"
-
+export PROCTOR_SENTRY_DSN="foo"
diff --git a/proctord/.env.test b/proctord/.env.test
@@ -29,4 +29,4 @@ export PROCTOR_MAIL_PASSWORD="password"
 export PROCTOR_MAIL_SERVER_HOST="smtp.mail.com"
 export PROCTOR_MAIL_SERVER_PORT="123"
 export PROCTOR_JOB_POD_ANNOTATIONS="{\"key.one\":\"true\"}"
-
+export PROCTOR_SENTRY_DSN="foo"
diff --git a/proctord/README.md b/proctord/README.md
@@ -60,4 +60,5 @@
 * `PROCTOR_MIN_CLIENT_VERSION` is minimum client version allowed to communicate with proctord
 * `PROCTOR_SCHEDULED_JOBS_FETCH_INTERVAL_IN_MINS` is the interval at which the scheduler fetches updated jobs from database
 * `PROCTOR_MAIL_USERNAME`, `PROCTOR_MAIL_PASSWORD`, `PROCTOR_MAIL_SERVER_HOST`, `PROCTOR_MAIL_SERVER_PORT` are the creds required to send notification to users on scheduled jobs execution
-* `PROCTOR_JOB_POD_ANNOTATIONS` is used to set any kubernetes pod specific annotations.
+* `PROCTOR_JOB_POD_ANNOTATIONS` is used to set any kubernetes pod specific annotations.
+* `PROCTOR_SENTRY_DSN` is used to set sentry DSN.
diff --git a/proctord/audit/auditor.go b/proctord/audit/auditor.go
@@ -1,6 +1,7 @@
 package audit
 
 import (
+	"github.com/getsentry/raven-go"
 	"github.com/gojektech/proctor/proctord/kubernetes"
 	"github.com/gojektech/proctor/proctord/logger"
 	"github.com/gojektech/proctor/proctord/storage"
@@ -39,16 +40,24 @@ func (auditor *auditor) JobsExecution(jobsExecutionAuditLog *postgres.JobsExecut
 	err := auditor.store.AuditJobsExecution(jobsExecutionAuditLog)
 	if err != nil {
 		logger.Error("Error auditing jobs execution", err)
+		raven.CaptureError(err, nil)
 	}
 }
 
 func (auditor *auditor) JobsExecutionStatus(jobExecutionID string) (string, error) {
 	status, err := auditor.kubeClient.JobExecutionStatus(jobExecutionID)
 	if err != nil {
 		logger.Error("Error getting job execution status", err)
+		raven.CaptureError(err, nil)
 		return "", err
 	}
 
 	err = auditor.store.UpdateJobsExecutionAuditLog(jobExecutionID, status)
+	if err != nil {
+		logger.Error("Error updating job status", err)
+		raven.CaptureError(err, nil)
+		return "", err
+	}
+
 	return status, err
 }
diff --git a/proctord/config/config.go b/proctord/config/config.go
@@ -140,3 +140,7 @@ func JobPodAnnotations() map[string]string {
 
 	return annotations
 }
+
+func SentryDSN() string {
+	return viper.GetString("SENTRY_DSN")
+}
diff --git a/proctord/config/config_test.go b/proctord/config/config_test.go
@@ -241,3 +241,11 @@ func TestJobPodAnnotations(t *testing.T) {
 
 	assert.Equal(t, map[string]string{"key.one": "true"}, JobPodAnnotations())
 }
+
+func TestSentryDSN(t *testing.T) {
+	os.Setenv("PROCTOR_SENTRY_DSN", "domain")
+
+	viper.AutomaticEnv()
+
+	assert.Equal(t, "domain", SentryDSN())
+}
diff --git a/proctord/glide.lock b/proctord/glide.lock
diff --git a/proctord/glide.yaml b/proctord/glide.yaml
@@ -1,6 +1,5 @@
 package: github.com/gojektech/proctor/proctord
 import:
-- package: github.com/Sirupsen/logrus
 - package: github.com/davecgh/go-spew
   subpackages:
   - spew
@@ -43,3 +42,5 @@ import:
 - package: github.com/robfig/cron
   version: v1.1
 - package: github.com/badoux/checkmail
+- package: github.com/getsentry/raven-go
+- package: github.com/sirupsen/logrus
diff --git a/proctord/jobs/execution/handler.go b/proctord/jobs/execution/handler.go
@@ -3,6 +3,7 @@ package execution
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/getsentry/raven-go"
 	"net/http"
 
 	"github.com/gojektech/proctor/proctord/audit"
@@ -41,6 +42,8 @@ func (handler *executionHandler) Status() http.HandlerFunc {
 			w.WriteHeader(http.StatusInternalServerError)
 			w.Write([]byte(utility.ServerError))
 			logger.Error(fmt.Sprintf("Error getting job status for job_id: %s", jobExecutionID), err.Error())
+			raven.CaptureError(err, map[string]string{"job_id": jobExecutionID})
+
 			return
 		}
 
@@ -68,6 +71,8 @@ func (handler *executionHandler) Handle() http.HandlerFunc {
 		defer req.Body.Close()
 		if err != nil {
 			logger.Error(fmt.Sprintf("User: %s: Error parsing request body", userEmail), err.Error())
+			raven.CaptureError(err, map[string]string{"user_email": userEmail})
+
 			jobsExecutionAuditLog.Errors = fmt.Sprintf("Error parsing request body: %s", err.Error())
 			jobsExecutionAuditLog.JobSubmissionStatus = utility.JobSubmissionClientError
 
@@ -79,6 +84,8 @@ func (handler *executionHandler) Handle() http.HandlerFunc {
 		jobExecutionID, err := handler.executioner.Execute(jobsExecutionAuditLog, job.Name, job.Args)
 		if err != nil {
 			logger.Error(fmt.Sprintf("%s: User %s: Error executing job: ", job.Name, userEmail), err.Error())
+			raven.CaptureError(err, map[string]string{"user_email": userEmail, "job_name": job.Name})
+
 			jobsExecutionAuditLog.Errors = fmt.Sprintf("Error executing job: %s", err.Error())
 			jobsExecutionAuditLog.JobSubmissionStatus = utility.JobSubmissionServerError
 

diff --git a/proctord/jobs/logs/handler.go b/proctord/jobs/logs/handler.go
@@ -2,6 +2,7 @@ package logs
 
 import (
 	"bufio"
+	"github.com/getsentry/raven-go"
 	"io"
 	"net/http"
 	"strings"
@@ -37,6 +38,7 @@ func CloseWebSocket(message string, conn *websocket.Conn) {
 	err := conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, message))
 	if err != nil {
 		_logger.Error("Error closing connection with client after logs are read")
+		raven.CaptureError(err, nil)
 	}
 	return
 }
@@ -46,6 +48,8 @@ func (l *logger) Stream() http.HandlerFunc {
 		conn, err := upgrader.Upgrade(w, req, nil)
 		if err != nil {
 			_logger.Error("Error upgrading connection to websocket protocol: ", err)
+			raven.CaptureError(err, nil)
+
 			w.WriteHeader(http.StatusBadRequest)
 			w.Write([]byte(utility.ClientError))
 			return
@@ -62,6 +66,8 @@ func (l *logger) Stream() http.HandlerFunc {
 		logStream, err := l.kubeClient.StreamJobLogs(jobName)
 		if err != nil {
 			_logger.Error("Error streaming logs from kube client: ", err)
+			raven.CaptureError(err, map[string]string{"job_name": jobName})
+
 			CloseWebSocket("Something went wrong", conn)
 			return
 		}
@@ -79,6 +85,8 @@ func (l *logger) Stream() http.HandlerFunc {
 				}
 
 				_logger.Error("Error reading from reader: ", err.Error())
+				raven.CaptureError(err, nil)
+
 				CloseWebSocket("Something went wrong", conn)
 				return
 			}
@@ -87,6 +95,8 @@ func (l *logger) Stream() http.HandlerFunc {
 			err = conn.WriteMessage(websocket.TextMessage, jobLogSingleLine[:])
 			if err != nil {
 				_logger.Error("Error writing logs to client: ", err)
+				raven.CaptureError(err, nil)
+
 				CloseWebSocket("Something went wrong", conn)
 				return
 			}

diff --git a/proctord/jobs/metadata/handler.go b/proctord/jobs/metadata/handler.go
@@ -2,6 +2,7 @@ package metadata
 
 import (
 	"encoding/json"
+	"github.com/getsentry/raven-go"
 	"net/http"
 
 	"github.com/gojektech/proctor/proctord/logger"
@@ -40,6 +41,7 @@ func (handler *handler) HandleSubmission() http.HandlerFunc {
 			err = handler.store.CreateOrUpdateJobMetadata(metadata)
 			if err != nil {
 				logger.Error("Error updating metadata", err.Error())
+				raven.CaptureError(err, nil)
 
 				w.WriteHeader(http.StatusInternalServerError)
 				w.Write([]byte(utility.ServerError))
@@ -57,6 +59,7 @@ func (handler *handler) HandleBulkDisplay() http.HandlerFunc {
 		jobMetadata, err := handler.store.GetAllJobsMetadata()
 		if err != nil {
 			logger.Error("Error fetching metadata", err.Error())
+			raven.CaptureError(err, nil)
 
 			w.WriteHeader(http.StatusInternalServerError)
 			w.Write([]byte(utility.ServerError))
@@ -66,6 +69,7 @@ func (handler *handler) HandleBulkDisplay() http.HandlerFunc {
 		jobsMetadataInJSON, err := json.Marshal(jobMetadata)
 		if err != nil {
 			logger.Error("Error marshalling jobs metadata in json", err.Error())
+			raven.CaptureError(err, nil)
 
 			w.WriteHeader(http.StatusInternalServerError)
 			w.Write([]byte(utility.ServerError))

diff --git a/proctord/jobs/schedule/handler.go b/proctord/jobs/schedule/handler.go
@@ -3,6 +3,7 @@ package schedule
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/getsentry/raven-go"
 	"github.com/gorilla/mux"
 	"net/http"
 	"strings"
@@ -43,6 +44,7 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc {
 		defer req.Body.Close()
 		if err != nil {
 			logger.Error("Error parsing request body for scheduling jobs: ", err.Error())
+			raven.CaptureError(err, nil)
 
 			w.WriteHeader(http.StatusBadRequest)
 			w.Write([]byte(utility.ClientError))
@@ -94,6 +96,7 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc {
 				w.Write([]byte(utility.NonExistentProcClientError))
 			} else {
 				logger.Error(fmt.Sprintf("Error fetching metadata for proc %s ", scheduledJob.Tags), scheduledJob.Name, err.Error())
+				raven.CaptureError(err, map[string]string{"job_tags": scheduledJob.Tags, "job_name": scheduledJob.Name})
 
 				w.WriteHeader(http.StatusInternalServerError)
 				w.Write([]byte(utility.ServerError))
@@ -107,13 +110,15 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc {
 		if err != nil {
 			if strings.Contains(err.Error(), "duplicate key value violates unique constraint") {
 				logger.Error(fmt.Sprintf("Client provided duplicate combination of scheduled job name and args: %s ", scheduledJob.Tags), scheduledJob.Name, scheduledJob.Args)
+				raven.CaptureError(err, map[string]string{"job_tags": scheduledJob.Tags, "job_name": scheduledJob.Name})
 
 				w.WriteHeader(http.StatusConflict)
 				w.Write([]byte(utility.DuplicateJobNameArgsClientError))
 
 				return
 			} else {
 				logger.Error(fmt.Sprintf("Error persisting scheduled job %s ", scheduledJob.Tags), scheduledJob.Name, err.Error())
+				raven.CaptureError(err, map[string]string{"job_tags": scheduledJob.Tags, "job_name": scheduledJob.Name})
 
 				w.WriteHeader(http.StatusInternalServerError)
 				w.Write([]byte(utility.ServerError))
@@ -124,7 +129,8 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc {
 
 		responseBody, err := json.Marshal(scheduledJob)
 		if err != nil {
-			logger.Error(fmt.Sprintf("Error marshaling response body %s ", scheduledJob.Tags),scheduledJob.Name, err.Error())
+			logger.Error(fmt.Sprintf("Error marshaling response body %s ", scheduledJob.Tags), scheduledJob.Name, err.Error())
+			raven.CaptureError(err, map[string]string{"job_tags": scheduledJob.Tags, "job_name": scheduledJob.Name})
 
 			w.WriteHeader(http.StatusInternalServerError)
 			w.Write([]byte(utility.ServerError))
@@ -143,6 +149,7 @@ func (scheduler *scheduler) GetScheduledJobs() http.HandlerFunc {
 		scheduledJobsStoreFormat, err := scheduler.store.GetEnabledScheduledJobs()
 		if err != nil {
 			logger.Error("Error fetching scheduled jobs", err.Error())
+			raven.CaptureError(err, nil)
 
 			w.WriteHeader(http.StatusInternalServerError)
 			w.Write([]byte(utility.ServerError))
@@ -151,15 +158,25 @@ func (scheduler *scheduler) GetScheduledJobs() http.HandlerFunc {
 
 		if len(scheduledJobsStoreFormat) == 0 {
 			logger.Error(utility.NoScheduledJobsError, nil)
+
 			w.WriteHeader(http.StatusNoContent)
 			return
 		}
 
-		scheduledJobs := FromStoreToHandler(scheduledJobsStoreFormat)
+		scheduledJobs, err := FromStoreToHandler(scheduledJobsStoreFormat)
+		if err != nil {
+			logger.Error("Error deserializing scheduled job args to map: ", err.Error())
+			raven.CaptureError(err, nil)
+
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(utility.ServerError))
+			return
+		}
 
 		scheduledJobsJson, err := json.Marshal(scheduledJobs)
 		if err != nil {
 			logger.Error("Error marshalling scheduled jobs", err.Error())
+			raven.CaptureError(err, nil)
 
 			w.WriteHeader(http.StatusInternalServerError)
 			w.Write([]byte(utility.ServerError))
@@ -182,6 +199,7 @@ func (scheduler *scheduler) GetScheduledJob() http.HandlerFunc {
 				return
 			}
 			logger.Error("Error fetching scheduled job", err.Error())
+			raven.CaptureError(err, nil)
 
 			w.WriteHeader(http.StatusInternalServerError)
 			w.Write([]byte(utility.ServerError))
@@ -196,11 +214,20 @@ func (scheduler *scheduler) GetScheduledJob() http.HandlerFunc {
 			return
 		}
 
-		job := GetScheduledJob(scheduledJob[0])
+		job, err := GetScheduledJob(scheduledJob[0])
+		if err != nil {
+			logger.Error("Error deserializing scheduled job args to map: ", err.Error())
+			raven.CaptureError(err, nil)
+
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(utility.ServerError))
+			return
+		}
 
 		scheduledJobJson, err := json.Marshal(job)
 		if err != nil {
 			logger.Error("Error marshalling scheduled job", err.Error())
+			raven.CaptureError(err, nil)
 
 			w.WriteHeader(http.StatusInternalServerError)
 			w.Write([]byte(utility.ServerError))
@@ -218,11 +245,13 @@ func (scheduler *scheduler) RemoveScheduledJob() http.HandlerFunc {
 		if err != nil {
 			if strings.Contains(err.Error(), "invalid input syntax") {
 				logger.Error(err.Error())
+
 				w.WriteHeader(http.StatusBadRequest)
 				w.Write([]byte("Invalid Job ID"))
 				return
 			}
 			logger.Error("Error fetching scheduled job", err.Error())
+			raven.CaptureError(err, nil)
 
 			w.WriteHeader(http.StatusInternalServerError)
 			w.Write([]byte(utility.ServerError))