feat(mlflow): Add s3 artifact type (#116)

* Add s3 artifact type and refactor common methods * Add s3 artifact service type as option in mlflow service * Update mocked artifact file * Update golang and linter versions * Fix linter comments * Remove deprecated linters * Fix broken artifact service interface * Update mocked artifact service * Rename unused argument * Add artifact client type field * Update mocked artifact service * Update s3 base endpoint based on env var * Refactor read artifact methods to return standardised notfound error * Reorder imports in artifact package * Fix incorrect logic for returning notfound error * Add additional unit tests * Fix lint comments * Address PR comments * Fix mlflow tests * Refactor s3 deletion steps * Refactor test methods to make them more readable * Remove codecov github action and configs temporarily
caraml-dev · Oct 10, 2024 · a98c895 · a98c895
1 parent 9b92012
commit a98c895
Show file tree

Hide file tree

Showing 28 changed files with 453 additions and 140 deletions.
diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml
@@ -21,8 +21,8 @@ on:
   workflow_dispatch:
 
 env:
-  GO_VERSION: "1.20"
-  GO_LINT_VERSION: v1.51.2
+  GO_VERSION: "1.22"
+  GO_LINT_VERSION: v1.58.1
   NODE_VERSION: 20
   ARTIFACT_RETENTION_DAYS: 7
   CONTAINER_REGISTRY: ghcr.io
@@ -102,7 +102,7 @@ jobs:
           cache-dependency-path: api/go.sum
 
       - name: Lint code
-        uses: golangci/golangci-lint-action@v2
+        uses: golangci/golangci-lint-action@v6
         with:
           version: ${{ env.GO_LINT_VERSION }}
           skip-go-installation: true
@@ -111,14 +111,6 @@ jobs:
       - name: Run Integration Test
         run: make it-test-api
 
-      - uses: codecov/codecov-action@v4
-        with:
-          flags: api-test
-          name: api-test
-          token: ${{ secrets.CODECOV_TOKEN }}
-          working-directory: ./api
-          codecov_yml_path: ../.github/workflows/codecov-config/codecov.yml
-
   e2e-test:
     runs-on: ubuntu-latest
     needs:

diff --git a/.github/workflows/codecov-config/codecov.yml b/.github/workflows/codecov-config/codecov.yml
diff --git a/.golangci.yml b/.golangci.yml
@@ -7,7 +7,6 @@ run:
 linters:
   enable:
     - bodyclose
-    - deadcode
     - errcheck
     - gocyclo
     - gofmt
@@ -19,9 +18,7 @@ linters:
     - misspell
     - revive
     - staticcheck
-    - structcheck
     - unused
-    - varcheck
 
 linters-settings:
   gocyclo:

diff --git a/Dockerfile b/Dockerfile
@@ -11,7 +11,7 @@ RUN yarn app build
 # ============================================================
 # Build stage 2: Build API
 # ============================================================
-FROM golang:1.20-alpine as go-builder
+FROM golang:1.22-alpine as go-builder
 WORKDIR /src/api
 COPY api api/
 COPY go.mod .

diff --git a/Makefile b/Makefile
@@ -16,7 +16,7 @@ all: setup init-dep lint test clean build run
 setup:
 	@echo "> Setting up tools..."
 	@test -x $(shell go env GOPATH)/bin/golangci-lint || \
-		curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/v1.53.3/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.53.3
+		curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/v1.58.1/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.58.1
 
 .PHONY: init-dep
 init-dep: init-dep-ui init-dep-api

diff --git a/api.Dockerfile b/api.Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.20-alpine as go-builder
+FROM golang:1.22-alpine as go-builder
 
 WORKDIR /src/api
 

diff --git a/api/api/projects_api.go b/api/api/projects_api.go
@@ -104,7 +104,7 @@ func (c *ProjectsController) UpdateProject(r *http.Request, vars map[string]stri
 	return Ok(updatedProject)
 }
 
-func (c *ProjectsController) GetProject(r *http.Request, vars map[string]string, body interface{}) *Response {
+func (c *ProjectsController) GetProject(_ *http.Request, vars map[string]string, _ interface{}) *Response {
 	projectID, _ := models.ParseID(vars["project_id"])
 	project, err := c.ProjectsService.FindByID(projectID)
 	if err != nil {

diff --git a/api/api/secrets_api.go b/api/api/secrets_api.go
@@ -14,7 +14,7 @@ type SecretsController struct {
 	*AppContext
 }
 
-func (c *SecretsController) GetSecret(r *http.Request, vars map[string]string, _ interface{}) *Response {
+func (c *SecretsController) GetSecret(_ *http.Request, vars map[string]string, _ interface{}) *Response {
 	projectID, _ := models.ParseID(vars["project_id"])
 	secretID, _ := models.ParseID(vars["secret_id"])
 	if projectID <= 0 || secretID <= 0 {
@@ -31,7 +31,7 @@ func (c *SecretsController) GetSecret(r *http.Request, vars map[string]string, _
 	return Ok(secret)
 }
 
-func (c *SecretsController) CreateSecret(r *http.Request, vars map[string]string, body interface{}) *Response {
+func (c *SecretsController) CreateSecret(_ *http.Request, vars map[string]string, body interface{}) *Response {
 	projectID, _ := models.ParseID(vars["project_id"])
 	_, err := c.ProjectsService.FindByID(projectID)
 	if err != nil {
@@ -56,7 +56,7 @@ func (c *SecretsController) CreateSecret(r *http.Request, vars map[string]string
 	return Created(secret)
 }
 
-func (c *SecretsController) UpdateSecret(r *http.Request, vars map[string]string, body interface{}) *Response {
+func (c *SecretsController) UpdateSecret(_ *http.Request, vars map[string]string, body interface{}) *Response {
 	updateRequest, ok := body.(*models.Secret)
 	if !ok {
 		return BadRequest("Invalid request body")
@@ -87,7 +87,7 @@ func (c *SecretsController) UpdateSecret(r *http.Request, vars map[string]string
 	return Ok(updatedSecret)
 }
 
-func (c *SecretsController) DeleteSecret(r *http.Request, vars map[string]string, _ interface{}) *Response {
+func (c *SecretsController) DeleteSecret(_ *http.Request, vars map[string]string, _ interface{}) *Response {
 	projectID, _ := models.ParseID(vars["project_id"])
 	secretID, _ := models.ParseID(vars["secret_id"])
 	if projectID <= 0 || secretID <= 0 {
@@ -102,7 +102,7 @@ func (c *SecretsController) DeleteSecret(r *http.Request, vars map[string]string
 	return NoContent()
 }
 
-func (c *SecretsController) ListSecret(r *http.Request, vars map[string]string, body interface{}) *Response {
+func (c *SecretsController) ListSecret(_ *http.Request, vars map[string]string, _ interface{}) *Response {
 	projectID, _ := models.ParseID(vars["project_id"])
 	_, err := c.ProjectsService.FindByID(projectID)
 	if err != nil {

diff --git a/api/cmd/bootstrap.go b/api/cmd/bootstrap.go
@@ -25,7 +25,7 @@ var (
 	bootstrapCmd        = &cobra.Command{
 		Use:   "bootstrap",
 		Short: "Start bootstrap job to populate Keto",
-		Run: func(cmd *cobra.Command, args []string) {
+		Run: func(_ *cobra.Command, _ []string) {
 			bootstrapConfig, err := loadBootstrapConfig(bootstrapConfigFile)
 			if err != nil {
 				log.Panicf("unable to load role members from input file: %v", err)

diff --git a/api/cmd/serve.go b/api/cmd/serve.go
@@ -28,7 +28,7 @@ var (
 	serveCmd    = &cobra.Command{
 		Use:   "serve",
 		Short: "Start MLP API server",
-		Run: func(cmd *cobra.Command, args []string) {
+		Run: func(_ *cobra.Command, _ []string) {
 			serveConfig, err := config.LoadAndValidate(configFiles...)
 			if err != nil {
 				log.Fatalf("failed initializing config: %v", err)
@@ -118,7 +118,7 @@ type uiEnvHandler struct {
 	LabelsBlacklist            []string              `json:"REACT_APP_LABELS_BLACKLIST"`
 }
 
-func (h uiEnvHandler) handler(w http.ResponseWriter, r *http.Request) {
+func (h uiEnvHandler) handler(w http.ResponseWriter, _ *http.Request) {
 	envJSON, err := json.Marshal(h)
 	if err != nil {
 		envJSON = []byte("{}")

diff --git a/api/it/database/database.go b/api/it/database/database.go
@@ -25,16 +25,19 @@ func connectionString(db string) string {
 }
 
 func create(conn *sql.DB, dbName string) (*sql.DB, error) {
+	var err error
 	if _, err := conn.Exec("CREATE DATABASE " + dbName); err != nil {
 		return nil, err
-	} else if testDb, err := sql.Open("postgres", connectionString(dbName)); err != nil {
+	}
+
+	var testDb *sql.DB
+	if testDb, err = sql.Open("postgres", connectionString(dbName)); err != nil {
 		if _, err := conn.Exec("DROP DATABASE " + dbName); err != nil {
 			log.Fatalf("Failed to cleanup integration test database: \n%s", err)
 		}
 		return nil, err
-	} else {
-		return testDb, nil
 	}
+	return testDb, nil
 }
 
 func migrate(db *sql.DB, dbName string) (*sql.DB, error) {
@@ -84,12 +87,14 @@ func CreateTestDatabase() (*gorm.DB, func(), error) {
 
 		cleanup()
 		return nil, nil, err
-	} else if gormDb, err := gorm.Open("postgres", testDb); err != nil {
+	}
+
+	var gormDb *gorm.DB
+	if gormDb, err = gorm.Open("postgres", testDb); err != nil {
 		cleanup()
 		return nil, nil, err
-	} else {
-		return gormDb, cleanup, nil
 	}
+	return gormDb, cleanup, nil
 }
 
 func WithTestDatabase(t *testing.T, test func(t *testing.T, db *gorm.DB)) {