Skip to content

Commit

Permalink
feat: add boostrap and serving command to mlp binary
Browse files Browse the repository at this point in the history
Signed-off-by: Khor Shu Heng <khor.heng@gojek.com>
  • Loading branch information
khorshuheng committed Jul 25, 2023
1 parent 21f31ed commit 050b931
Show file tree
Hide file tree
Showing 11 changed files with 135 additions and 17 deletions.
3 changes: 2 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ COPY api api/
COPY go.mod .
COPY go.sum .
COPY db-migrations ./db-migrations
RUN go build -o bin/mlp-api ./api/cmd/main.go
RUN go build -o bin/mlp-api ./api/main.go

# ============================================================
# Build stage 3: Run the app
Expand All @@ -28,3 +28,4 @@ COPY --from=go-builder /src/api/bin/mlp-api /usr/bin/mlp
COPY --from=go-builder /src/api/db-migrations ./db-migrations

ENTRYPOINT ["sh", "-c", "mlp \"$@\"", "--"]
CMD ["serve"]
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ build-ui: clean-ui
.PHONY: build-api
build-api: clean-bin
@echo "> Building API binary ..."
@cd ${API_PATH} && go build -o ../bin/${BIN_NAME} ./cmd/main.go
@cd ${API_PATH} && go build -o ../bin/${BIN_NAME} main.go

.PHONY: build-api-image
build-api-image: version
Expand All @@ -111,7 +111,7 @@ build-image: version
.PHONY: run
run: local-env
@echo "> Running application ..."
@go run api/cmd/main.go --config config-dev.yaml
@go run api/main.go serve --config config-dev.yaml

.PHONY: start-ui
start-ui:
Expand Down
5 changes: 3 additions & 2 deletions api.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,13 @@ COPY api api/
COPY go.mod .
COPY go.sum .

RUN go build -o bin/mlp-api ./api/cmd/main.go
RUN go build -o bin/mlp-api ./api/main.go

# Clean image with mlp-api binary
FROM alpine:3.16

COPY --from=go-builder /src/api/bin/mlp-api /usr/bin/mlp
COPY db-migrations ./db-migrations

ENTRYPOINT ["sh", "-c", "mlp \"$@\"", "--"]
ENTRYPOINT ["sh", "-c", "mlp \"$@\"", "--"]
CMD ["serve"]
54 changes: 54 additions & 0 deletions api/cmd/bootstrap.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package cmd

import (
"context"

"github.com/spf13/cobra"

"github.com/caraml-dev/mlp/api/config"
"github.com/caraml-dev/mlp/api/log"
"github.com/caraml-dev/mlp/api/pkg/authz/enforcer"
)

type BootstrapOptions struct {
ProjectReaders []string
MLPAdmins []string
}

var (
bootstrapOpts = &BootstrapOptions{}
bootstrapCmd = &cobra.Command{
Use: "bootstrap",
Short: "Start bootstrap job to populate Keto",
Run: func(cmd *cobra.Command, args []string) {
err := startKetoBootstrap(globalConfig, bootstrapOpts)
if err != nil {
log.Panicf("unable to bootstrap keto: %v", err)
}
},
}
)

func init() {
bootstrapCmd.Flags().StringSliceVarP(&bootstrapOpts.ProjectReaders, "project-readers", "r",
[]string{}, "Comma separated list of project readers")
bootstrapCmd.Flags().StringSliceVar(&bootstrapOpts.MLPAdmins, "mlp-admins", []string{},
"Comma separated list of MLP admins")
}

func startKetoBootstrap(globalCfg *config.Config, bootstrapOpts *BootstrapOptions) error {
authEnforcer, err := enforcer.NewEnforcerBuilder().
KetoEndpoints(globalCfg.Authorization.KetoRemoteRead, globalCfg.Authorization.KetoRemoteWrite).
Build()
if err != nil {
return err
}
updateRequest := enforcer.NewAuthorizationUpdateRequest()
updateRequest.UpdateRoleMembers("mlp.projects.reader", bootstrapOpts.ProjectReaders)
updateRequest.UpdateRoleMembers("mlp.admin", bootstrapOpts.MLPAdmins)
err = authEnforcer.UpdateAuthorization(context.Background(), updateRequest)
if err != nil {
return err
}
return nil
}
43 changes: 43 additions & 0 deletions api/cmd/root.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package cmd

import (
"github.com/spf13/cobra"

"github.com/caraml-dev/mlp/api/config"
"github.com/caraml-dev/mlp/api/log"
)

var (
configFiles []string
globalConfig *config.Config
rootCmd = &cobra.Command{
Use: "mlp",
Short: "CaraML Machine Learning Platform Console",
Long: "CaraML Machine Learning Platform Console, which provides a web UI to interact with different CaraML " +
"services.",
}
)

func init() {
cobra.OnInitialize(initConfig)
rootCmd.PersistentFlags().StringSliceVarP(&configFiles, "config", "c", []string{},
"Comma separated list of config files to load. The last config file will take precedence over the "+
"previous ones.")
rootCmd.AddCommand(serveCmd)
rootCmd.AddCommand(bootstrapCmd)
}

func initConfig() {
var err error
globalConfig, err = config.LoadAndValidate(configFiles...)
if err != nil {
log.Fatalf("failed initializing config: %v", err)
}
}

func Execute() {
err := rootCmd.Execute()
if err != nil {
log.Fatalf("failed executing root command: %v", err)
}
}
20 changes: 11 additions & 9 deletions api/cmd/main.go → api/cmd/serve.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package main
package cmd

import (
"encoding/json"
Expand All @@ -13,7 +13,7 @@ import (
"github.com/gorilla/mux"
"github.com/heptiolabs/healthcheck"
"github.com/rs/cors"
flag "github.com/spf13/pflag"
"github.com/spf13/cobra"

"github.com/caraml-dev/mlp/api/api"
apiV2 "github.com/caraml-dev/mlp/api/api/v2"
Expand All @@ -23,15 +23,17 @@ import (
"github.com/caraml-dev/mlp/api/pkg/authz/enforcer"
)

func main() {
configFiles := flag.StringSliceP("config", "c", []string{}, "Path to a configuration files")
flag.Parse()

cfg, err := config.LoadAndValidate(*configFiles...)
if err != nil {
log.Panicf("failed initializing config: %v", err)
var (
serveCmd = &cobra.Command{
Use: "serve",
Short: "Start MLP API server",
Run: func(cmd *cobra.Command, args []string) {
startServer(globalConfig)
},
}
)

func startServer(cfg *config.Config) {
// init db
db, err := database.InitDB(cfg.Database)
if err != nil {
Expand Down
9 changes: 9 additions & 0 deletions api/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package main

import (
"github.com/caraml-dev/mlp/api/cmd"
)

func main() {
cmd.Execute()
}
1 change: 1 addition & 0 deletions docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ services:
env_file: .env.development
environment:
- DATABASE_HOST=postgres
restart: on-failure

postgres:
image: bitnami/postgresql:14.5.0
Expand Down
1 change: 1 addition & 0 deletions full.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@ FROM ${MLP_API_IMAGE}
COPY ui/build ./ui/build

ENTRYPOINT ["sh", "-c", "mlp \"$@\"", "--"]
CMD ["serve"]
4 changes: 3 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ require (
github.com/prometheus/client_golang v1.11.1
github.com/prometheus/client_model v0.2.0
github.com/rs/cors v1.7.0
github.com/spf13/cobra v1.7.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.8.1
github.com/uber/jaeger-client-go v2.16.0+incompatible
Expand All @@ -39,7 +40,6 @@ require (
golang.org/x/oauth2 v0.5.0
golang.org/x/sync v0.1.0
google.golang.org/api v0.106.0
google.golang.org/grpc v1.52.3
gopkg.in/errgo.v2 v2.1.0
gopkg.in/yaml.v2 v2.4.0
k8s.io/client-go v0.26.0
Expand Down Expand Up @@ -84,6 +84,7 @@ require (
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/imdario/mergo v0.3.6 // indirect
github.com/imkira/go-interpol v1.1.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.15.0 // indirect
Expand Down Expand Up @@ -128,6 +129,7 @@ require (
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230131230820-1c016267d619 // indirect
google.golang.org/grpc v1.52.3 // indirect
google.golang.org/protobuf v1.29.0 // indirect
gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect
gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
Expand Down
8 changes: 6 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
Expand Down Expand Up @@ -406,6 +407,8 @@ github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk=
github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
Expand Down Expand Up @@ -567,8 +570,6 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
github.com/ory/keto-client-go v0.11.0-alpha.0 h1:CJyKa6DhiYVDDtHa0DR//wkhLH8SXgWIXLZUugOYwH8=
github.com/ory/keto-client-go v0.11.0-alpha.0/go.mod h1:z/TmfbuoIU3DAHiv5a+cTyHXYc5mQDx38Ve8g2kYI00=
github.com/ory/keto/proto v0.11.1-alpha.0 h1:xVpFRnnIAGGvP9lYIUwjSWmrO7qVoLn20bT6NxzYQy4=
github.com/ory/keto/proto v0.11.1-alpha.0/go.mod h1:M9J/kybmyLKRmvvSqYzmRVYx2avY3yDMdUPinsck1q0=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
Expand Down Expand Up @@ -624,6 +625,7 @@ github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
Expand All @@ -641,6 +643,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
Expand Down

0 comments on commit 050b931

Please sign in to comment.