From 6e9e44dfadf13c3899707beda09d92b4f907e24d Mon Sep 17 00:00:00 2001 From: Jinjing Zhou Date: Tue, 5 Jul 2022 16:01:33 +0800 Subject: [PATCH] feat: Support specify build target (#497) * support specify function Signed-off-by: Jinjing.Zhou * change example Signed-off-by: Jinjing.Zhou * fix Signed-off-by: Jinjing.Zhou * fix test Signed-off-by: Jinjing.Zhou * fix naming Signed-off-by: Jinjing.Zhou * fix lint Signed-off-by: Jinjing.Zhou * fix Signed-off-by: Jinjing.Zhou --- examples/mnist/README.md | 2 +- examples/mnist/build_gpu.envd | 13 ------------- pkg/app/build.go | 14 +++++++++----- pkg/app/up.go | 14 +++++++++----- pkg/builder/builder.go | 6 ++++-- pkg/builder/builder_test.go | 1 + pkg/builder/util.go | 24 ++++++++++++++++++++++++ 7 files changed, 48 insertions(+), 26 deletions(-) delete mode 100644 examples/mnist/build_gpu.envd diff --git a/examples/mnist/README.md b/examples/mnist/README.md index 88b09b3e0..b84950e40 100644 --- a/examples/mnist/README.md +++ b/examples/mnist/README.md @@ -13,7 +13,7 @@ Then you can open jupyter notebook at [`http://localhost:8888`](http://localhost ## Quick start (GPU) ```bash -$ envd up -f build_gpu.envd +$ envd up -f :build_gpu # Call build_gpu function in build.envd ``` Also you can use `-f` option to specify the file to build diff --git a/examples/mnist/build_gpu.envd b/examples/mnist/build_gpu.envd deleted file mode 100644 index fe4af3691..000000000 --- a/examples/mnist/build_gpu.envd +++ /dev/null @@ -1,13 +0,0 @@ -def build(): - base(os="ubuntu20.04", language="python3") - install.vscode_extensions([ - "ms-python.python", - ]) - #config.pip_index(url = "https://pypi.tuna.tsinghua.edu.cn/simple") - install.python_packages([ - "tensorflow", - "numpy", - ]) - install.cuda(version="11.6", cudnn="8") - shell("zsh") - config.jupyter(password="", port=8888) diff --git a/pkg/app/build.go b/pkg/app/build.go index a383c6f89..e4d05adf3 100644 --- a/pkg/app/build.go +++ b/pkg/app/build.go @@ -48,10 +48,10 @@ To build and push the image to a registry: DefaultText: "PROJECT:dev", }, &cli.PathFlag{ - Name: "file", - Usage: "Name of the build.envd", + Name: "from", + Usage: "Function to execute, format `file:func`", Aliases: []string{"f"}, - Value: "build.envd", + Value: "build.envd:build", }, &cli.PathFlag{ Name: "path", @@ -82,7 +82,11 @@ func build(clicontext *cli.Context) error { return errors.Wrap(err, "failed to get absolute path of the build context") } - manifest, err := filepath.Abs(filepath.Join(buildContext, clicontext.Path("file"))) + filename, funcname, err := builder.ParseFromStr(clicontext.String("from")) + if err != nil { + return err + } + manifest, err := filepath.Abs(filepath.Join(buildContext, filename)) if err != nil { return errors.Wrap(err, "failed to get absolute path of the build file") } @@ -111,7 +115,7 @@ func build(clicontext *cli.Context) error { "output": output, }).Debug("starting build command") builder, err := builder.New(clicontext.Context, cfg, - manifest, buildContext, tag, output, debug) + manifest, funcname, buildContext, tag, output, debug) if err != nil { return errors.Wrap(err, "failed to create the builder") } diff --git a/pkg/app/up.go b/pkg/app/up.go index a5034c1a9..f19adfcef 100644 --- a/pkg/app/up.go +++ b/pkg/app/up.go @@ -61,10 +61,10 @@ var CommandUp = &cli.Command{ Aliases: []string{"v"}, }, &cli.PathFlag{ - Name: "file", - Usage: "Name of the build.envd", + Name: "from", + Usage: "Function to execute, format `file:func`", Aliases: []string{"f"}, - Value: "build.envd", + Value: "build.envd:build", }, &cli.PathFlag{ Name: "private-key", @@ -105,8 +105,12 @@ func up(clicontext *cli.Context) error { if err != nil { return errors.Wrap(err, "failed to get absolute path of the build context") } + filename, funcname, err := builder.ParseFromStr(clicontext.String("from")) + if err != nil { + return err + } - manifest, err := filepath.Abs(filepath.Join(buildContext, clicontext.Path("file"))) + manifest, err := filepath.Abs(filepath.Join(buildContext, filename)) if err != nil { return errors.Wrap(err, "failed to get absolute path of the build file") } @@ -137,7 +141,7 @@ func up(clicontext *cli.Context) error { logger.Debug("starting up command") debug := clicontext.Bool("debug") output := "" - builder, err := builder.New(clicontext.Context, config, manifest, + builder, err := builder.New(clicontext.Context, config, manifest, funcname, buildContext, tag, output, debug) if err != nil { return errors.Wrap(err, "failed to create the builder") diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go index 60a13e617..1c26a133c 100644 --- a/pkg/builder/builder.go +++ b/pkg/builder/builder.go @@ -50,6 +50,7 @@ type generalBuilder struct { progressMode string tag string buildContextDir string + buildfuncname string entries []client.ExportEntry @@ -58,7 +59,7 @@ type generalBuilder struct { buildkitd.Client } -func New(ctx context.Context, configFilePath, manifestFilePath, +func New(ctx context.Context, configFilePath, manifestFilePath, funcname, buildContextDir, tag string, output string, debug bool) (Builder, error) { entries, err := parseOutput(output) if err != nil { @@ -84,6 +85,7 @@ func New(ctx context.Context, configFilePath, manifestFilePath, b := &generalBuilder{ manifestFilePath: manifestFilePath, + buildfuncname: funcname, configFilePath: configFilePath, buildContextDir: buildContextDir, entries: entries, @@ -141,7 +143,7 @@ func (b generalBuilder) interpret() error { return errors.Wrap(err, "failed to exec starlark file") } - if _, err := b.ExecFile(b.manifestFilePath, "build"); err != nil { + if _, err := b.ExecFile(b.manifestFilePath, b.buildfuncname); err != nil { return errors.Wrap(err, "failed to exec starlark file") } return nil diff --git a/pkg/builder/builder_test.go b/pkg/builder/builder_test.go index 0affeaf12..821f6fd76 100644 --- a/pkg/builder/builder_test.go +++ b/pkg/builder/builder_test.go @@ -55,6 +55,7 @@ var _ = Describe("Builder", func() { configFilePath: configFilePath, progressMode: "auto", tag: tag, + buildfuncname: "build", logger: logrus.WithFields(logrus.Fields{ "tag": tag, }), diff --git a/pkg/builder/util.go b/pkg/builder/util.go index 46d790a24..8e14320f4 100644 --- a/pkg/builder/util.go +++ b/pkg/builder/util.go @@ -29,6 +29,11 @@ import ( "github.com/sirupsen/logrus" ) +const ( + defaultFile = "build.envd" + defaultFunc = "build" +) + func ImageConfigStr(labels map[string]string, ports map[string]struct{}, entrypoint []string) (string, error) { pl := platforms.Normalize(platforms.DefaultSpec()) @@ -172,3 +177,22 @@ func resolveExporterDest(exporter, dest string) (func(map[string]string) (io.Wri return nil, "", nil } } + +func ParseFromStr(fromStr string) (string, string, error) { + filename := defaultFile + funcname := defaultFunc + if strings.Contains(fromStr, ":") { + fromArr := strings.Split(fromStr, ":") + + if len(fromArr) != 2 { + return "", "", errors.New("invalid from format, expected `file:func`") + } + if fromArr[0] != "" { + filename = fromArr[0] + } + if fromArr[1] != "" { + funcname = fromArr[1] + } + } + return filename, funcname, nil +}