feat: implement persistence

fixes #2
fixes #4
fixes #7

Author: vbehar

.gitignore

@@ -1,7 +1,8 @@
 .DS_Store
 .idea/
 
-build/
-dist/
+/build/
+/dist/
+/data/
 
 .cr-release-packages/

charts/lighthouse-webui-plugin/templates/deployment.yaml

@@ -11,8 +11,11 @@ metadata:
   annotations: {{- tpl (toYaml .) $ | trim | nindent 4 }}
   {{- end }}
 spec:
-  replicas: {{ .Values.deployment.replicas }}
+  replicas: 1
   revisionHistoryLimit: {{ .Values.deployment.revisionHistoryLimit }}
+  {{- with .Values.deployment.strategy }}
+  strategy: {{- tpl (toYaml .) $ | trim | nindent 4 }}
+  {{- end }}
   selector:
     matchLabels: {{- include "webui.labels.selector" . | nindent 6 }}
   template:
@@ -57,6 +60,12 @@ spec:
         - -log-level
         - {{ . }}
         {{- end }}
+        - -store-data-path
+        - "/data"
+        - -store-max-events
+        - {{ .Values.config.store.gc.maxEventsToKeep }}
+        - -store-events-max-age
+        - {{ .Values.config.store.gc.eventsMaxAge }}
         env:
         - name: XDG_CONFIG_HOME
           value: /home/jenkins
@@ -71,6 +80,9 @@ spec:
         envFrom:
           {{- toYaml . | trim | nindent 10 }}
         {{- end }}
+        volumeMounts:
+        - name: data
+          mountPath: "/data"
         ports:
         - name: http
           containerPort: 8080
@@ -84,6 +96,14 @@ spec:
         {{- with .Values.pod.resources }}
         resources: {{- toYaml . | trim | nindent 10 }}
         {{- end }}
+      volumes:
+      - name: data
+        {{- if .Values.persistence.enabled }}
+        persistentVolumeClaim:
+          claimName: {{ include "webui.fullname" . }}
+        {{- else }}
+        emptyDir: {}
+        {{- end }}
       {{- with .Values.pod.securityContext }}
       securityContext: {{- toYaml . | trim | nindent 8 }}
       {{- end }}

charts/lighthouse-webui-plugin/templates/pvc.yaml

@@ -0,0 +1,25 @@
+{{- if .Values.persistence.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ template "webui.fullname" . }}
+  labels:
+    {{- include "webui.labels" . | nindent 4 }}
+    {{- with .Values.persistence.labels }}
+      {{- tpl (toYaml .) $ | trim | nindent 4 }}
+    {{- end }}
+  {{- with .Values.persistence.annotations }}
+  annotations: {{- tpl (toYaml .) $ | trim | nindent 4 }}
+  {{- end }}
+spec:
+  {{- with .Values.persistence.storageClassName }}
+  storageClassName: {{ . | quote }}
+  {{- end }}
+  accessModes:
+    {{- range .Values.persistence.accessModes }}
+    - {{ . | quote }}
+    {{- end }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.size | quote }}
+{{- end -}}

charts/lighthouse-webui-plugin/values.yaml

@@ -11,6 +11,13 @@ config:
   namespace: jx
   resyncInterval: 60s
   logLevel: INFO
+  store:
+    gc:
+      # max number of events to keep in the store - if non-zero
+      maxEventsToKeep: 0
+      # max age of the events to keep in the store - if non-zero
+      # this is a golang duration. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+      eventsMaxAge: 0
 
 secrets:
   lighthouse:
@@ -26,10 +33,12 @@ image:
   pullPolicy:
 
 deployment:
-  replicas: 1
   revisionHistoryLimit: 2
   labels: {}
   annotations: {}
+  strategy:
+    # if you enable persistence, you should switch to `Recreate`
+    type: RollingUpdate
 
 pod:
   resources:
@@ -53,6 +62,7 @@ pod:
     fsGroup: 1000
   env: {}
   envFrom: []
+
 service:
   port: 80
   type:
@@ -96,10 +106,16 @@ istio:
   apiVersion: networking.istio.io/v1beta1
   gateway: jx-gateway
 
-jx:
-  # whether to create a Release CRD when installing charts with Release CRDs included
-  releaseCRD: true
-  
+# persistence for the events
+persistence:
+  enabled: false
+  size: 1Gi
+  accessModes:
+    - ReadWriteOnce
+  storageClassName:
+  labels: {}
+  annotations: {} 
+
 serviceAccount:
   # allow additional annotations to be added to the ServiceAccount
   # such as for workload identity on clouds
@@ -110,3 +126,7 @@ role:
   - apiGroups: ["lighthouse.jenkins.io"]
     resources: ["lighthousejobs"]
     verbs: ["list", "watch", "get"]
+
+jx:
+  # whether to create a Release CRD when installing charts with Release CRDs included
+  releaseCRD: true

cmd/server/main.go

@@ -7,6 +7,9 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"os/signal"
+	"sync"
+	"syscall"
 	"time"
 
 	webui "github.com/jenkins-x-plugins/lighthouse-webui-plugin"
@@ -27,6 +30,7 @@ var (
 		keeperEndpoint        string
 		keeperSyncInterval    time.Duration
 		eventTraceURLTemplate string
+		storeConfig           webui.StoreConfig
 		kubeConfigPath        string
 		listenAddr            string
 		logLevel              string
@@ -42,6 +46,9 @@ func init() {
 	flag.DurationVar(&options.keeperSyncInterval, "keeper-sync-interval", 1*time.Minute, "Interval to poll the Lighthouse Keeper service for its state")
 	flag.StringVar(&options.eventTraceURLTemplate, "event-trace-url-template", "", "Go template string used to build the event trace URL")
 	flag.StringVar(&options.logLevel, "log-level", "INFO", "Log level - one of: trace, debug, info, warn(ing), error, fatal or panic")
+	flag.StringVar(&options.storeConfig.DataPath, "store-data-path", "", "If non-empty, the events store will be persisted on disk in the directory")
+	flag.IntVar(&options.storeConfig.MaxEvents, "store-max-events", 0, "If non-zero, the internal GC will ensure that no more than that many number of events will be stored/persisted")
+	flag.DurationVar(&options.storeConfig.EventsMaxAge, "store-events-max-age", 0, "If non-zero, the internal GC will ensure to events older than this age (duration) will be removed from the store")
 	flag.StringVar(&options.kubeConfigPath, "kubeconfig", kube.DefaultKubeConfigPath(), "Kubernetes Config Path. Default: KUBECONFIG env var value")
 	flag.StringVar(&options.listenAddr, "listen-addr", ":8080", "Address on which the server will listen for incoming connections")
 	flag.BoolVar(&options.printVersion, "version", false, "Print the version")
@@ -55,8 +62,8 @@ func main() {
 		return
 	}
 
-	ctx, cancelFunc := context.WithCancel(context.Background())
-	defer cancelFunc()
+	ctx, cancelCtx := context.WithCancel(context.Background())
+	defer cancelCtx()
 
 	logger := logrus.New()
 	logLevel, err := logrus.ParseLevel(options.logLevel)
@@ -65,7 +72,7 @@ func main() {
 	} else {
 		logger.SetLevel(logLevel)
 	}
-	logger.WithField("logLevel", logLevel).Info("Starting")
+	logger.WithField("logLevel", logLevel).WithField("version", version.Version).Info("Starting")
 
 	kConfig, err := kube.NewConfig(options.kubeConfigPath)
 	if err != nil {
@@ -76,11 +83,12 @@ func main() {
 		logger.WithError(err).Fatal("failed to create a Lighthouse client")
 	}
 
-	store, err := webui.NewStore()
+	store, err := webui.NewStore(options.storeConfig, logger)
 	if err != nil {
 		logger.WithError(err).Fatal("failed to create a new store")
 	}
 
+	logger.WithField("endpoint", options.keeperEndpoint).WithField("syncInterval", options.keeperSyncInterval).Info("Starting Keeper Syncer")
 	(&webui.KeeperSyncer{
 		KeeperEndpoint: options.keeperEndpoint,
 		SyncInterval:   options.keeperSyncInterval,
@@ -116,11 +124,45 @@ func main() {
 	if err != nil {
 		logger.WithError(err).Fatal("failed to initialize the HTTP handler")
 	}
-	http.Handle("/", handler)
-
-	logger.WithField("listenAddr", options.listenAddr).Info("Starting HTTP Server")
-	err = http.ListenAndServe(options.listenAddr, nil)
-	if !errors.Is(err, http.ErrServerClosed) {
-		logger.WithError(err).Fatal("failed to start HTTP server")
+	httpServer := http.Server{
+		Handler: handler,
+		Addr:    options.listenAddr,
 	}
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		<-ctx.Done()
+		shutdownCtx, cancelShutdownFunc := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancelShutdownFunc()
+
+		logger.Info("Shutting down the HTTP server...")
+		if err := httpServer.Shutdown(shutdownCtx); err != nil {
+			logger.Warn("Timed-out while waiting for the HTTP server to shutdown!")
+		}
+
+		logger.Info("Closing the store...")
+		if err := store.Close(); err != nil {
+			logger.WithError(err).Warn("failed to close the store")
+		}
+
+		wg.Done()
+	}()
+	go func() {
+		logger.WithField("listenAddr", options.listenAddr).Info("Starting HTTP Server")
+		err = httpServer.ListenAndServe()
+		if !errors.Is(err, http.ErrServerClosed) {
+			logger.WithError(err).Fatal("failed to start HTTP server")
+		}
+	}()
+
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
+	sig := <-c
+
+	logger.WithField("signal", sig).Info("Interrupted by user (or killed) !")
+	cancelCtx()
+	wg.Wait()
+
+	logger.Info("This is the end, my friend")
 }

event.go

@@ -5,7 +5,14 @@ import (
 	"time"
 )
 
-type Events []Event
+type Events struct {
+	Events []Event
+	Counts struct {
+		Kinds        map[string]int
+		Repositories map[string]int
+		Senders      map[string]int
+	}
+}
 
 type Event struct {
 	GUID       string

job.go

@@ -8,7 +8,15 @@ import (
 	lhv1alpha1 "github.com/jenkins-x/lighthouse/pkg/apis/lighthouse/v1alpha1"
 )
 
-type Jobs []Job
+type Jobs struct {
+	Jobs   []Job
+	Counts struct {
+		States       map[string]int
+		Types        map[string]int
+		Repositories map[string]int
+		Authors      map[string]int
+	}
+}
 
 type Job struct {
 	Name        string
@@ -19,6 +27,7 @@ type Job struct {
 	Branch      string
 	Build       string
 	Context     string
+	Author      string
 	State       string
 	Description string
 	ReportURL   string
@@ -55,6 +64,13 @@ func JobFromLighthouseJob(lhjob *lhv1alpha1.LighthouseJob) Job {
 		j.End = lhjob.Status.CompletionTime.Time
 		j.Duration = j.End.Sub(j.Start)
 	}
+	if lhjob.Spec.Refs != nil {
+		for _, pr := range lhjob.Spec.Refs.Pulls {
+			if pr.Author != "" && j.Author == "" {
+				j.Author = pr.Author
+			}
+		}
+	}
 	return j
 }