From 050b93184c06ee470762c3a8a922437ec2d9c4c8 Mon Sep 17 00:00:00 2001 From: Khor Shu Heng Date: Tue, 25 Jul 2023 01:09:28 +0800 Subject: [PATCH] feat: add boostrap and serving command to mlp binary Signed-off-by: Khor Shu Heng --- Dockerfile | 3 +- Makefile | 4 +-- api.Dockerfile | 5 ++-- api/cmd/bootstrap.go | 54 +++++++++++++++++++++++++++++++++++ api/cmd/root.go | 43 ++++++++++++++++++++++++++++ api/cmd/{main.go => serve.go} | 20 +++++++------ api/main.go | 9 ++++++ docker-compose.yaml | 1 + full.Dockerfile | 1 + go.mod | 4 ++- go.sum | 8 ++++-- 11 files changed, 135 insertions(+), 17 deletions(-) create mode 100644 api/cmd/bootstrap.go create mode 100644 api/cmd/root.go rename api/cmd/{main.go => serve.go} (94%) create mode 100644 api/main.go diff --git a/Dockerfile b/Dockerfile index f5d681a0..915b46e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ COPY api api/ COPY go.mod . COPY go.sum . COPY db-migrations ./db-migrations -RUN go build -o bin/mlp-api ./api/cmd/main.go +RUN go build -o bin/mlp-api ./api/main.go # ============================================================ # Build stage 3: Run the app @@ -28,3 +28,4 @@ COPY --from=go-builder /src/api/bin/mlp-api /usr/bin/mlp COPY --from=go-builder /src/api/db-migrations ./db-migrations ENTRYPOINT ["sh", "-c", "mlp \"$@\"", "--"] +CMD ["serve"] \ No newline at end of file diff --git a/Makefile b/Makefile index 7287b0bd..922d4627 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ build-ui: clean-ui .PHONY: build-api build-api: clean-bin @echo "> Building API binary ..." - @cd ${API_PATH} && go build -o ../bin/${BIN_NAME} ./cmd/main.go + @cd ${API_PATH} && go build -o ../bin/${BIN_NAME} main.go .PHONY: build-api-image build-api-image: version @@ -111,7 +111,7 @@ build-image: version .PHONY: run run: local-env @echo "> Running application ..." - @go run api/cmd/main.go --config config-dev.yaml + @go run api/main.go serve --config config-dev.yaml .PHONY: start-ui start-ui: diff --git a/api.Dockerfile b/api.Dockerfile index e371b26c..ad0369ca 100644 --- a/api.Dockerfile +++ b/api.Dockerfile @@ -10,7 +10,7 @@ COPY api api/ COPY go.mod . COPY go.sum . -RUN go build -o bin/mlp-api ./api/cmd/main.go +RUN go build -o bin/mlp-api ./api/main.go # Clean image with mlp-api binary FROM alpine:3.16 @@ -18,4 +18,5 @@ FROM alpine:3.16 COPY --from=go-builder /src/api/bin/mlp-api /usr/bin/mlp COPY db-migrations ./db-migrations -ENTRYPOINT ["sh", "-c", "mlp \"$@\"", "--"] \ No newline at end of file +ENTRYPOINT ["sh", "-c", "mlp \"$@\"", "--"] +CMD ["serve"] diff --git a/api/cmd/bootstrap.go b/api/cmd/bootstrap.go new file mode 100644 index 00000000..5e6609ea --- /dev/null +++ b/api/cmd/bootstrap.go @@ -0,0 +1,54 @@ +package cmd + +import ( + "context" + + "github.com/spf13/cobra" + + "github.com/caraml-dev/mlp/api/config" + "github.com/caraml-dev/mlp/api/log" + "github.com/caraml-dev/mlp/api/pkg/authz/enforcer" +) + +type BootstrapOptions struct { + ProjectReaders []string + MLPAdmins []string +} + +var ( + bootstrapOpts = &BootstrapOptions{} + bootstrapCmd = &cobra.Command{ + Use: "bootstrap", + Short: "Start bootstrap job to populate Keto", + Run: func(cmd *cobra.Command, args []string) { + err := startKetoBootstrap(globalConfig, bootstrapOpts) + if err != nil { + log.Panicf("unable to bootstrap keto: %v", err) + } + }, + } +) + +func init() { + bootstrapCmd.Flags().StringSliceVarP(&bootstrapOpts.ProjectReaders, "project-readers", "r", + []string{}, "Comma separated list of project readers") + bootstrapCmd.Flags().StringSliceVar(&bootstrapOpts.MLPAdmins, "mlp-admins", []string{}, + "Comma separated list of MLP admins") +} + +func startKetoBootstrap(globalCfg *config.Config, bootstrapOpts *BootstrapOptions) error { + authEnforcer, err := enforcer.NewEnforcerBuilder(). + KetoEndpoints(globalCfg.Authorization.KetoRemoteRead, globalCfg.Authorization.KetoRemoteWrite). + Build() + if err != nil { + return err + } + updateRequest := enforcer.NewAuthorizationUpdateRequest() + updateRequest.UpdateRoleMembers("mlp.projects.reader", bootstrapOpts.ProjectReaders) + updateRequest.UpdateRoleMembers("mlp.admin", bootstrapOpts.MLPAdmins) + err = authEnforcer.UpdateAuthorization(context.Background(), updateRequest) + if err != nil { + return err + } + return nil +} diff --git a/api/cmd/root.go b/api/cmd/root.go new file mode 100644 index 00000000..861f6f8e --- /dev/null +++ b/api/cmd/root.go @@ -0,0 +1,43 @@ +package cmd + +import ( + "github.com/spf13/cobra" + + "github.com/caraml-dev/mlp/api/config" + "github.com/caraml-dev/mlp/api/log" +) + +var ( + configFiles []string + globalConfig *config.Config + rootCmd = &cobra.Command{ + Use: "mlp", + Short: "CaraML Machine Learning Platform Console", + Long: "CaraML Machine Learning Platform Console, which provides a web UI to interact with different CaraML " + + "services.", + } +) + +func init() { + cobra.OnInitialize(initConfig) + rootCmd.PersistentFlags().StringSliceVarP(&configFiles, "config", "c", []string{}, + "Comma separated list of config files to load. The last config file will take precedence over the "+ + "previous ones.") + rootCmd.AddCommand(serveCmd) + rootCmd.AddCommand(bootstrapCmd) +} + +func initConfig() { + var err error + globalConfig, err = config.LoadAndValidate(configFiles...) + if err != nil { + log.Fatalf("failed initializing config: %v", err) + } +} + +func Execute() { + err := rootCmd.Execute() + if err != nil { + log.Fatalf("failed executing root command: %v", err) + } +} diff --git a/api/cmd/main.go b/api/cmd/serve.go similarity index 94% rename from api/cmd/main.go rename to api/cmd/serve.go index 46725818..f26d1ebe 100644 --- a/api/cmd/main.go +++ b/api/cmd/serve.go @@ -1,4 +1,4 @@ -package main +package cmd import ( "encoding/json" @@ -13,7 +13,7 @@ import ( "github.com/gorilla/mux" "github.com/heptiolabs/healthcheck" "github.com/rs/cors" - flag "github.com/spf13/pflag" + "github.com/spf13/cobra" "github.com/caraml-dev/mlp/api/api" apiV2 "github.com/caraml-dev/mlp/api/api/v2" @@ -23,15 +23,17 @@ import ( "github.com/caraml-dev/mlp/api/pkg/authz/enforcer" ) -func main() { - configFiles := flag.StringSliceP("config", "c", []string{}, "Path to a configuration files") - flag.Parse() - - cfg, err := config.LoadAndValidate(*configFiles...) - if err != nil { - log.Panicf("failed initializing config: %v", err) +var ( + serveCmd = &cobra.Command{ + Use: "serve", + Short: "Start MLP API server", + Run: func(cmd *cobra.Command, args []string) { + startServer(globalConfig) + }, } +) +func startServer(cfg *config.Config) { // init db db, err := database.InitDB(cfg.Database) if err != nil { diff --git a/api/main.go b/api/main.go new file mode 100644 index 00000000..5828f1e3 --- /dev/null +++ b/api/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/caraml-dev/mlp/api/cmd" +) + +func main() { + cmd.Execute() +} diff --git a/docker-compose.yaml b/docker-compose.yaml index 5aa6ea67..2a0f700b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -12,6 +12,7 @@ services: env_file: .env.development environment: - DATABASE_HOST=postgres + restart: on-failure postgres: image: bitnami/postgresql:14.5.0 diff --git a/full.Dockerfile b/full.Dockerfile index 3ed0ff2c..df4e040c 100644 --- a/full.Dockerfile +++ b/full.Dockerfile @@ -4,3 +4,4 @@ FROM ${MLP_API_IMAGE} COPY ui/build ./ui/build ENTRYPOINT ["sh", "-c", "mlp \"$@\"", "--"] +CMD ["serve"] diff --git a/go.mod b/go.mod index 6303b456..d06d5917 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/prometheus/client_golang v1.11.1 github.com/prometheus/client_model v0.2.0 github.com/rs/cors v1.7.0 + github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.1 github.com/uber/jaeger-client-go v2.16.0+incompatible @@ -39,7 +40,6 @@ require ( golang.org/x/oauth2 v0.5.0 golang.org/x/sync v0.1.0 google.golang.org/api v0.106.0 - google.golang.org/grpc v1.52.3 gopkg.in/errgo.v2 v2.1.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/client-go v0.26.0 @@ -84,6 +84,7 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/imkira/go-interpol v1.1.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.15.0 // indirect @@ -128,6 +129,7 @@ require ( golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230131230820-1c016267d619 // indirect + google.golang.org/grpc v1.52.3 // indirect google.golang.org/protobuf v1.29.0 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/go-playground/assert.v1 v1.2.1 // indirect diff --git a/go.sum b/go.sum index 457d3a85..184b3bca 100644 --- a/go.sum +++ b/go.sum @@ -132,6 +132,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= @@ -406,6 +407,8 @@ github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= @@ -567,8 +570,6 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/ory/keto-client-go v0.11.0-alpha.0 h1:CJyKa6DhiYVDDtHa0DR//wkhLH8SXgWIXLZUugOYwH8= github.com/ory/keto-client-go v0.11.0-alpha.0/go.mod h1:z/TmfbuoIU3DAHiv5a+cTyHXYc5mQDx38Ve8g2kYI00= -github.com/ory/keto/proto v0.11.1-alpha.0 h1:xVpFRnnIAGGvP9lYIUwjSWmrO7qVoLn20bT6NxzYQy4= -github.com/ory/keto/proto v0.11.1-alpha.0/go.mod h1:M9J/kybmyLKRmvvSqYzmRVYx2avY3yDMdUPinsck1q0= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= @@ -624,6 +625,7 @@ github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= @@ -641,6 +643,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=