Feature/persitence (#3)

* - updated go modules
- improved docker file
- improved ui styling
- added sqlite persistence with ent
- added dbimporter tool to import records
- regenreated ui api
- fixed mailtransformer
- removed obsolete data file
- used different dockerfile as go baseimage, enabled CGO
- added import to reveal app

.vscode/launch.json

@@ -21,6 +21,16 @@
             "mode": "auto",
             "program": "cmd/mailtransformer/main.go"
         },
+        {
+            "name": "Launch dbimport Go Package",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "cmd/dbimport/main.go",
+            "args": [
+                "${workspaceFolder}/configs/debug/"                
+            ]
+        },
         {
             "type": "firefox",
             "request": "launch",

README.md

@@ -0,0 +1,10 @@
+# reval
+
+## Get started quickly:
+
+1. checkout repository
+2. create folder _tmp_
+3. place files _Llama-2-13b-chat-hf_responses.json_ and _vicuna-33b-v1.3_responses.json_ into _tmp_ folder
+4. rename the files to _import_Llama-2-13b-chat-hf_responses.json_ and _import_vicuna-33b-v1.3_responses.json_
+5. run ```docker compose up```
+6. open http://localhost:8080

api/evaluationapi.yaml

@@ -112,8 +112,7 @@ components:
       description: The result of the current evaluation.
       properties:
         id:
-          type: integer
-          format: int32
+          type: string
           description: Unique id of the message evaluation.
           example: 1234
         evaluationResult:
@@ -130,8 +129,7 @@ components:
       description: The data of the next evaluation.
       properties:
         id:
-          type: integer
-          format: int32
+          type: string
           description: Unique id of the evaluation.
           example: 1234
         response:

build/reval/Dockerfile

@@ -1,44 +1,60 @@
 # syntax=docker/dockerfile:1
 
+# --------------------------------------------
 # Build the go backend application from source
-FROM golang:1.20 AS go-build-stage
+# --------------------------------------------
+FROM golang:1.20-bullseye AS go-build-stage
 
 WORKDIR /app
 
 COPY go.mod go.sum ./
 RUN go mod download && go mod verify
 
+#copy all repo files
 COPY . .
 
-RUN CGO_ENABLED=0 GOOS=linux go build -v -o /usr/local/bin/app/ ./... 
+# create code for OpenApi spec
+RUN go install github.com/deepmap/oapi-codegen/cmd/oapi-codegen@v1.13.4
 
+# generate open api spec for go server
+RUN $GOPATH/bin/oapi-codegen -config configs/docker/evaluationapi.cfg.yaml api/evaluationapi.yaml
+# build application
+RUN CGO_ENABLED=1 GOOS=linux go build -v -o /usr/local/bin/app/ ./... 
 
+# -----------------------------------------
 # Run the go backend tests in the container
+# -----------------------------------------
 FROM go-build-stage AS go-run-test-stage
 RUN go test -v ./...
 
+
+# --------------------------------------------------
+# Build the Angular frontend application from source
+# --------------------------------------------------
 FROM node:20.5-alpine AS node-build-stage
 
 WORKDIR /app
 
 COPY ui/reval-web/package.json ui/reval-web/package-lock.json ./
 
-RUN apk update && apk add openjdk17-jdk && npm install @openapitools/openapi-generator-cli && npm install
+RUN apk update && apk add openjdk17-jre && npm install @openapitools/openapi-generator-cli && npm install
 COPY . .
 RUN cd ui/reval-web/ && npm run build
 
-# Deploy the application binary into a lean image
+# -----------------------------------------------------------------
+# Deploy the build outputs and all required files into a lean image
+# -----------------------------------------------------------------
 FROM gcr.io/distroless/base-debian11 AS build-release-stage
 
-ENV GIN_MODE=release
 
 WORKDIR /app
 
 COPY --from=go-build-stage /usr/local/bin/app/ /app/configs/docker/reval.env /app/
-#COPY --from=go-build-stage /app/configs/docker/reval.env /app
-COPY --from=go-build-stage /app/data /app/data
 COPY --from=node-build-stage /app/ui/reval-web/dist/reval-web/ /app/www
 
+# make sure to run webserver in release mode
+ENV GIN_MODE=release
+
 EXPOSE 8080
 
 USER nonroot:nonroot

cmd/dbimport/main.go

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"github.com/rherlt/reval/internal/config"
+	"github.com/rherlt/reval/internal/data"
+	"github.com/rherlt/reval/internal/persistence"
+)
+
+func main() {
+
+	config.Configure()
+	persistence.SetupDb()
+	data.ImportData()
+	persistence.CloseClient()
+}

cmd/mailtransformer/main.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strconv"
 	"strings"
 
 	"github.com/rherlt/reval/internal/api/evaluationapi"
@@ -45,7 +46,7 @@ func main() {
 	for i := 0; i < len(mails); i++ {
 		var currentMail = mails[i]
 		transformedMails = append(transformedMails, evaluationapi.GetEvaluationResponse{
-			Id: int32(i),
+			Id: strconv.Itoa(i),
 			Request: evaluationapi.Message{
 				Body:    GetStringAfterInBetween(currentMail.RequestMail, "Betreff: ", "\n\n"),
 				From:    "chatgpt/gpt-3.5-turbo",

cmd/reval/main.go

@@ -3,53 +3,59 @@ package main
 import (
 	"log"
 	"net/http"
-	"os"
 
 	"github.com/rherlt/reval/internal/api/evaluationapi"
 	"github.com/rherlt/reval/internal/config"
 	"github.com/rherlt/reval/internal/controller"
+	"github.com/rherlt/reval/internal/data"
+	"github.com/rherlt/reval/internal/persistence"
 
 	"github.com/gin-contrib/cors"
 	"github.com/gin-gonic/gin"
 )
 
 func main() {
 
-	//load config path from command line or use "." (current application path)
-	var configPath string = "."
-	if len(os.Args) > 1 {
-		configPath = os.Args[1:][0]
-	}
-
-	err := config.LoadConfig(configPath)
-	if err != nil {
-		log.Fatal("cannot load config:", err)
-	}
-
-	//todo replace with load from database
-	controller.LoadDataFromFile()
+	config.Configure()
 
 	//setup gin webserver
 	r := gin.Default()
 
+	//setup http redirect from app root / to web base url, e.g. /ui/
 	r.NoRoute(func(c *gin.Context) {
 		c.Redirect(http.StatusMovedPermanently, config.Current.Gin_Web_BaseUrl)
 	})
 
 	//register folder for static web deployment
 	r.Static("/ui/", config.Current.Gin_Web_Path)
 
+	//setup CORS
 	corsConfig := cors.DefaultConfig()
 	corsConfig.AllowHeaders = append(corsConfig.AllowHeaders, config.Current.Gin_Cors_AdditionalAllowedHeaders...)
 	corsConfig.AllowAllOrigins = config.Current.Gin_Cors_AllowAllOrigins
 	r.Use(cors.New(corsConfig))
-	//TODO: https://github.com/deepmap/oapi-codegen/blob/master/examples/petstore-expanded/gin/petstore.go#L21C1-L48C1
-	r.GET("/swagger/openapi.json", controller.GetSwagger)
 
-	//register HTTP handlers
+	//register HTTP handlers for evaluatio api
 	si := new(controller.EvaluationApiServerInterface)
 	evaluationapi.RegisterHandlersWithOptions(r, si, si.GetServerOptions())
 
+	importRequired := !persistence.DbExistis()
+
+	//setup database
+	err := persistence.SetupDb()
+	defer persistence.CloseClient()
+
+	if err != nil {
+		log.Fatal("cannot setup database: ", err)
+	}
+
+	if importRequired {
+		err = data.ImportData()
+		if err != nil {
+			log.Fatal("cannot import data: ", err)
+		}
+	}
+
 	//run webserver
 	r.Run(config.Current.Gin_WebServerAddress)
 }

configs/debug/reval.env

@@ -1,9 +1,11 @@
 # default configuration for debugging reval
-DATAPATH=../../data/reval-transformed.json
 # Gin Webserver Settings
 GIN_CORS_ALLOWALLORIGINS=true
 GIN_CORS_ADDITIONALALLOWEDHEADERS=Authorization
 GIN_WEB_PATH=../../ui/reval-web/dist/reval-web/
 GIN_WEBSERVERADDRESS=:8080
 GIN_API_BASEURL=/api/
-GIN_WEB_BASEURL=/ui/
+GIN_WEB_BASEURL=/ui/
+DB_TYPE=sqlite
+DB_SQLITE_CONNECTION=file:../../tmp/reval.db?cache=shared&_fk=1
+DATA_IMPORT_GLOB=../../tmp/import_*.json

configs/docker/evaluationapi.cfg.yaml

@@ -0,0 +1,6 @@
+package: evaluationapi
+generate:
+  gin-server: true
+  embedded-spec: true
+  models: true
+output: internal/api/evaluationapi/server.gen.go

configs/docker/reval.env

@@ -1,9 +1,11 @@
 # default configuration for debugging reval
-DATAPATH=./data/reval-transformed.json
 # Gin Webserver Settings
 GIN_CORS_ALLOWALLORIGINS=true
 GIN_CORS_ADDITIONALALLOWEDHEADERS=Authorization
 GIN_WEB_PATH=./www/
 GIN_WEBSERVERADDRESS=:8080
 GIN_API_BASEURL=/api/
 GIN_WEB_BASEURL=/ui/
+DB_TYPE=sqlite
+DB_SQLITE_CONNECTION=file:./data/reval.db?cache=shared&_fk=1
+DATA_IMPORT_GLOB=./data/import_*.json

docker-compose.yaml

@@ -6,4 +6,6 @@ services:
 
     image: local/reval:latest
     ports:
-      - "8080:8080"
+      - "8080:8080"
+    volumes:
+      - "./tmp/:/app/data/" #sqlite database path