Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(mlflow): Add s3 artifact type #116

Merged
merged 22 commits into from
Oct 10, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
5c36417
Add s3 artifact type and refactor common methods
deadlycoconuts Sep 9, 2024
d85b94b
Add s3 artifact service type as option in mlflow service
deadlycoconuts Sep 9, 2024
1764bab
Update mocked artifact file
deadlycoconuts Sep 9, 2024
f54c70c
Update golang and linter versions
deadlycoconuts Sep 9, 2024
f5a004d
Fix linter comments
deadlycoconuts Sep 9, 2024
a31085c
Remove deprecated linters
deadlycoconuts Sep 9, 2024
0ae9356
Fix broken artifact service interface
deadlycoconuts Sep 9, 2024
3061360
Update mocked artifact service
deadlycoconuts Sep 9, 2024
68b8f79
Rename unused argument
deadlycoconuts Sep 9, 2024
6c3a172
Add artifact client type field
deadlycoconuts Sep 11, 2024
6b280fe
Update mocked artifact service
deadlycoconuts Sep 11, 2024
b040244
Update s3 base endpoint based on env var
deadlycoconuts Sep 12, 2024
232c6a0
Refactor read artifact methods to return standardised notfound error
deadlycoconuts Sep 16, 2024
5eb4cb9
Reorder imports in artifact package
deadlycoconuts Sep 16, 2024
d94d925
Fix incorrect logic for returning notfound error
deadlycoconuts Sep 17, 2024
a7348b6
Add additional unit tests
deadlycoconuts Sep 18, 2024
5a49a32
Fix lint comments
deadlycoconuts Sep 18, 2024
ce9ebe6
Address PR comments
deadlycoconuts Sep 24, 2024
730abdf
Fix mlflow tests
deadlycoconuts Sep 24, 2024
1ffe047
Refactor s3 deletion steps
deadlycoconuts Sep 24, 2024
f6e5f3e
Refactor test methods to make them more readable
deadlycoconuts Sep 30, 2024
dc25762
Remove codecov github action and configs temporarily
deadlycoconuts Oct 1, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 3 additions & 11 deletions .github/workflows/ci-pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ on:
workflow_dispatch:

env:
GO_VERSION: "1.20"
GO_LINT_VERSION: v1.51.2
GO_VERSION: "1.22"
GO_LINT_VERSION: v1.58.1
NODE_VERSION: 20
ARTIFACT_RETENTION_DAYS: 7
CONTAINER_REGISTRY: ghcr.io
Expand Down Expand Up @@ -102,7 +102,7 @@ jobs:
cache-dependency-path: api/go.sum

- name: Lint code
uses: golangci/golangci-lint-action@v2
uses: golangci/golangci-lint-action@v6
with:
version: ${{ env.GO_LINT_VERSION }}
skip-go-installation: true
Expand All @@ -111,14 +111,6 @@ jobs:
- name: Run Integration Test
run: make it-test-api

- uses: codecov/codecov-action@v4
with:
flags: api-test
name: api-test
token: ${{ secrets.CODECOV_TOKEN }}
working-directory: ./api
codecov_yml_path: ../.github/workflows/codecov-config/codecov.yml

e2e-test:
runs-on: ubuntu-latest
needs:
Expand Down
5 changes: 0 additions & 5 deletions .github/workflows/codecov-config/codecov.yml

This file was deleted.

3 changes: 0 additions & 3 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ run:
linters:
enable:
- bodyclose
- deadcode
deadlycoconuts marked this conversation as resolved.
Show resolved Hide resolved
- errcheck
- gocyclo
- gofmt
Expand All @@ -19,9 +18,7 @@ linters:
- misspell
- revive
- staticcheck
- structcheck
- unused
- varcheck

linters-settings:
gocyclo:
Expand Down
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ RUN yarn app build
# ============================================================
# Build stage 2: Build API
# ============================================================
FROM golang:1.20-alpine as go-builder
FROM golang:1.22-alpine as go-builder
WORKDIR /src/api
COPY api api/
COPY go.mod .
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ all: setup init-dep lint test clean build run
setup:
@echo "> Setting up tools..."
@test -x $(shell go env GOPATH)/bin/golangci-lint || \
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/v1.53.3/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.53.3
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/v1.58.1/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.58.1

.PHONY: init-dep
init-dep: init-dep-ui init-dep-api
Expand Down
2 changes: 1 addition & 1 deletion api.Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM golang:1.20-alpine as go-builder
FROM golang:1.22-alpine as go-builder

WORKDIR /src/api

Expand Down
2 changes: 1 addition & 1 deletion api/api/projects_api.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ func (c *ProjectsController) UpdateProject(r *http.Request, vars map[string]stri
return Ok(updatedProject)
}

func (c *ProjectsController) GetProject(r *http.Request, vars map[string]string, body interface{}) *Response {
func (c *ProjectsController) GetProject(_ *http.Request, vars map[string]string, _ interface{}) *Response {
projectID, _ := models.ParseID(vars["project_id"])
project, err := c.ProjectsService.FindByID(projectID)
if err != nil {
Expand Down
10 changes: 5 additions & 5 deletions api/api/secrets_api.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ type SecretsController struct {
*AppContext
}

func (c *SecretsController) GetSecret(r *http.Request, vars map[string]string, _ interface{}) *Response {
func (c *SecretsController) GetSecret(_ *http.Request, vars map[string]string, _ interface{}) *Response {
projectID, _ := models.ParseID(vars["project_id"])
secretID, _ := models.ParseID(vars["secret_id"])
if projectID <= 0 || secretID <= 0 {
Expand All @@ -31,7 +31,7 @@ func (c *SecretsController) GetSecret(r *http.Request, vars map[string]string, _
return Ok(secret)
}

func (c *SecretsController) CreateSecret(r *http.Request, vars map[string]string, body interface{}) *Response {
func (c *SecretsController) CreateSecret(_ *http.Request, vars map[string]string, body interface{}) *Response {
projectID, _ := models.ParseID(vars["project_id"])
_, err := c.ProjectsService.FindByID(projectID)
if err != nil {
Expand All @@ -56,7 +56,7 @@ func (c *SecretsController) CreateSecret(r *http.Request, vars map[string]string
return Created(secret)
}

func (c *SecretsController) UpdateSecret(r *http.Request, vars map[string]string, body interface{}) *Response {
func (c *SecretsController) UpdateSecret(_ *http.Request, vars map[string]string, body interface{}) *Response {
updateRequest, ok := body.(*models.Secret)
if !ok {
return BadRequest("Invalid request body")
Expand Down Expand Up @@ -87,7 +87,7 @@ func (c *SecretsController) UpdateSecret(r *http.Request, vars map[string]string
return Ok(updatedSecret)
}

func (c *SecretsController) DeleteSecret(r *http.Request, vars map[string]string, _ interface{}) *Response {
func (c *SecretsController) DeleteSecret(_ *http.Request, vars map[string]string, _ interface{}) *Response {
projectID, _ := models.ParseID(vars["project_id"])
secretID, _ := models.ParseID(vars["secret_id"])
if projectID <= 0 || secretID <= 0 {
Expand All @@ -102,7 +102,7 @@ func (c *SecretsController) DeleteSecret(r *http.Request, vars map[string]string
return NoContent()
}

func (c *SecretsController) ListSecret(r *http.Request, vars map[string]string, body interface{}) *Response {
func (c *SecretsController) ListSecret(_ *http.Request, vars map[string]string, _ interface{}) *Response {
projectID, _ := models.ParseID(vars["project_id"])
_, err := c.ProjectsService.FindByID(projectID)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion api/cmd/bootstrap.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ var (
bootstrapCmd = &cobra.Command{
Use: "bootstrap",
Short: "Start bootstrap job to populate Keto",
Run: func(cmd *cobra.Command, args []string) {
Run: func(_ *cobra.Command, _ []string) {
bootstrapConfig, err := loadBootstrapConfig(bootstrapConfigFile)
if err != nil {
log.Panicf("unable to load role members from input file: %v", err)
Expand Down
4 changes: 2 additions & 2 deletions api/cmd/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ var (
serveCmd = &cobra.Command{
Use: "serve",
Short: "Start MLP API server",
Run: func(cmd *cobra.Command, args []string) {
Run: func(_ *cobra.Command, _ []string) {
serveConfig, err := config.LoadAndValidate(configFiles...)
if err != nil {
log.Fatalf("failed initializing config: %v", err)
Expand Down Expand Up @@ -118,7 +118,7 @@ type uiEnvHandler struct {
LabelsBlacklist []string `json:"REACT_APP_LABELS_BLACKLIST"`
}

func (h uiEnvHandler) handler(w http.ResponseWriter, r *http.Request) {
func (h uiEnvHandler) handler(w http.ResponseWriter, _ *http.Request) {
envJSON, err := json.Marshal(h)
if err != nil {
envJSON = []byte("{}")
Expand Down
17 changes: 11 additions & 6 deletions api/it/database/database.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,19 @@ func connectionString(db string) string {
}

func create(conn *sql.DB, dbName string) (*sql.DB, error) {
var err error
if _, err := conn.Exec("CREATE DATABASE " + dbName); err != nil {
return nil, err
} else if testDb, err := sql.Open("postgres", connectionString(dbName)); err != nil {
}

var testDb *sql.DB
if testDb, err = sql.Open("postgres", connectionString(dbName)); err != nil {
if _, err := conn.Exec("DROP DATABASE " + dbName); err != nil {
log.Fatalf("Failed to cleanup integration test database: \n%s", err)
}
return nil, err
} else {
return testDb, nil
}
return testDb, nil
}

func migrate(db *sql.DB, dbName string) (*sql.DB, error) {
Expand Down Expand Up @@ -84,12 +87,14 @@ func CreateTestDatabase() (*gorm.DB, func(), error) {

cleanup()
return nil, nil, err
} else if gormDb, err := gorm.Open("postgres", testDb); err != nil {
}

var gormDb *gorm.DB
if gormDb, err = gorm.Open("postgres", testDb); err != nil {
cleanup()
return nil, nil, err
} else {
return gormDb, cleanup, nil
}
return gormDb, cleanup, nil
}

func WithTestDatabase(t *testing.T, test func(t *testing.T, db *gorm.DB)) {
Expand Down
Loading
Loading