diff --git a/contrib/nydusify/cmd/nydusify.go b/contrib/nydusify/cmd/nydusify.go index c05966d371d..6c6a8df7027 100644 --- a/contrib/nydusify/cmd/nydusify.go +++ b/contrib/nydusify/cmd/nydusify.go @@ -112,6 +112,35 @@ func getCacheReference(c *cli.Context, target string) (string, error) { return cache, nil } +func getPrefetchPatterns(c *cli.Context) (string, error) { + prefetchedDir := c.String("prefetch-dir") + prefetchPatterns := c.Bool("prefetch-patterns") + + if len(prefetchedDir) > 0 && prefetchPatterns { + return "", fmt.Errorf("--prefetch-dir conflicts with --prefetch-patterns") + } + + var patterns string + + if prefetchPatterns { + bytes, err := ioutil.ReadAll(os.Stdin) + if err != nil { + return "", errors.Wrap(err, "read prefetch patterns from STDIN") + } + patterns = string(bytes) + } + + if len(prefetchedDir) > 0 { + patterns = prefetchedDir + } + + if len(patterns) <= 0 { + patterns = "/" + } + + return patterns, nil +} + func main() { logrus.SetFormatter(&logrus.TextFormatter{ FullTimestamp: true, @@ -139,7 +168,8 @@ func main() { &cli.BoolFlag{Name: "source-insecure", Required: false, Usage: "Allow http/insecure source registry communication", EnvVars: []string{"SOURCE_INSECURE"}}, &cli.BoolFlag{Name: "target-insecure", Required: false, Usage: "Allow http/insecure target registry communication", EnvVars: []string{"TARGET_INSECURE"}}, &cli.StringFlag{Name: "work-dir", Value: "./tmp", Usage: "Work directory path for image conversion", EnvVars: []string{"WORK_DIR"}}, - &cli.StringFlag{Name: "prefetch-dir", Value: "/", Usage: "Prefetch directory for nydus image, use absolute path of rootfs", EnvVars: []string{"PREFETCH_DIR"}}, + &cli.StringFlag{Name: "prefetch-dir", Value: "", Usage: "Prefetched directory for nydus image, use absolute path of rootfs", EnvVars: []string{"PREFETCH_DIR"}}, + &cli.BoolFlag{Name: "prefetch-patterns", Value: false, Usage: "Prefetched file path patterns from STDIN, specify absolute/relative path of rootfs line by line", EnvVars: []string{"PREFETCH_PATTERNS"}}, &cli.StringFlag{Name: "nydus-image", Value: "nydus-image", Usage: "The nydus-image binary path, if unset, search in PATH environment", EnvVars: []string{"NYDUS_IMAGE"}}, &cli.BoolFlag{Name: "multi-platform", Value: false, Usage: "Merge OCI & Nydus manifest to manifest index for target image, please ensure that OCI manifest already exists in target image", EnvVars: []string{"MULTI_PLATFORM"}}, &cli.StringFlag{Name: "platform", Value: "linux/" + runtime.GOARCH, Usage: "Let nydusify choose image of specified platform from manifest index. Possible value is `amd64` or `arm64`"}, @@ -238,6 +268,11 @@ func main() { return err } + prefetchPatterns, err := getPrefetchPatterns(c) + if err != nil { + return err + } + opt := converter.Opt{ Logger: logger, SourceProviders: sourceProviders, @@ -248,11 +283,11 @@ func main() { CacheMaxRecords: cacheMaxRecords, CacheVersion: cacheVersion, - WorkDir: c.String("work-dir"), - PrefetchDir: c.String("prefetch-dir"), - NydusImagePath: c.String("nydus-image"), - MultiPlatform: c.Bool("multi-platform"), - DockerV2Format: c.Bool("docker-v2-format"), + WorkDir: c.String("work-dir"), + PrefetchPatterns: prefetchPatterns, + NydusImagePath: c.String("nydus-image"), + MultiPlatform: c.Bool("multi-platform"), + DockerV2Format: c.Bool("docker-v2-format"), BackendType: backendType, BackendConfig: backendConfig, diff --git a/contrib/nydusify/examples/converter/main.go b/contrib/nydusify/examples/converter/main.go index 8859111dfc2..f9860b04f10 100644 --- a/contrib/nydusify/examples/converter/main.go +++ b/contrib/nydusify/examples/converter/main.go @@ -50,11 +50,11 @@ func main() { SourceProviders: sourceProviders, TargetRemote: targetRemote, - WorkDir: wordDir, - PrefetchDir: "/", - NydusImagePath: nydusImagePath, - MultiPlatform: false, - DockerV2Format: true, + WorkDir: wordDir, + PrefetchPatterns: "/", + NydusImagePath: nydusImagePath, + MultiPlatform: false, + DockerV2Format: true, } cvt, err := converter.New(opt) diff --git a/contrib/nydusify/pkg/build/builder.go b/contrib/nydusify/pkg/build/builder.go index 959414425b6..77ec769d0ba 100644 --- a/contrib/nydusify/pkg/build/builder.go +++ b/contrib/nydusify/pkg/build/builder.go @@ -20,9 +20,9 @@ type BuilderOption struct { RootfsPath string BackendType string BackendConfig string - PrefetchDir string WhiteoutSpec string OutputJSONPath string + PrefetchPatterns string // A regular file or fifo into which commands nydus-image to dump contents. BlobPath string AlignedChunk bool @@ -52,22 +52,13 @@ func NewBuilder(binaryPath string) *Builder { } } -func (builder *Builder) run(args []string, stdinInfo ...string) error { +func (builder *Builder) run(args []string, prefetchPatterns string) error { logrus.Debugf("\tCommand: %s %s", builder.binaryPath, strings.Join(args[:], " ")) cmd := exec.Command(builder.binaryPath, args...) cmd.Stdout = builder.stdout cmd.Stderr = builder.stderr - - stdin, err := cmd.StdinPipe() - if err != nil { - return err - } - - for _, s := range stdinInfo { - io.WriteString(stdin, s) - } - stdin.Close() + cmd.Stdin = strings.NewReader(prefetchPatterns) if err := cmd.Run(); err != nil { logrus.WithError(err).Errorf("fail to run %v %+v", builder.binaryPath, args) @@ -93,7 +84,7 @@ func (builder *Builder) Compact(option CompactOption) error { if option.ChunkDict != "" { args = append(args, "--chunk-dict", option.ChunkDict) } - return builder.run(args) + return builder.run(args, "") } // Run exec nydus-image CLI to build layer @@ -132,9 +123,9 @@ func (builder *Builder) Run(option BuilderOption) error { option.RootfsPath, ) - if option.PrefetchDir != "" { + if len(option.PrefetchPatterns) > 0 { args = append(args, "--prefetch-policy", "fs") } - return builder.run(args, option.PrefetchDir) + return builder.run(args, option.PrefetchPatterns) } diff --git a/contrib/nydusify/pkg/build/workflow.go b/contrib/nydusify/pkg/build/workflow.go index 587bffbcccd..243d9cd5f1b 100644 --- a/contrib/nydusify/pkg/build/workflow.go +++ b/contrib/nydusify/pkg/build/workflow.go @@ -17,10 +17,10 @@ import ( ) type WorkflowOption struct { - ChunkDict string - TargetDir string - NydusImagePath string - PrefetchDir string + ChunkDict string + TargetDir string + NydusImagePath string + PrefetchPatterns string } type Workflow struct { @@ -88,10 +88,6 @@ func NewWorkflow(option WorkflowOption) (*Workflow, error) { backendConfig := fmt.Sprintf(`{"dir": "%s"}`, blobsDir) builder := NewBuilder(option.NydusImagePath) - if option.PrefetchDir == "" { - option.PrefetchDir = "/" - } - return &Workflow{ WorkflowOption: option, blobsDir: blobsDir, @@ -116,7 +112,7 @@ func (workflow *Workflow) Build( ParentBootstrapPath: workflow.parentBootstrapPath, BootstrapPath: workflow.bootstrapPath, RootfsPath: layerDir, - PrefetchDir: workflow.PrefetchDir, + PrefetchPatterns: workflow.PrefetchPatterns, WhiteoutSpec: whiteoutSpec, OutputJSONPath: workflow.buildOutputJSONPath(), BlobPath: blobPath, diff --git a/contrib/nydusify/pkg/converter/converter.go b/contrib/nydusify/pkg/converter/converter.go index 77dc790fef3..5505fe9e3c5 100644 --- a/contrib/nydusify/pkg/converter/converter.go +++ b/contrib/nydusify/pkg/converter/converter.go @@ -74,9 +74,9 @@ type Opt struct { CacheMaxRecords uint CacheVersion string - NydusImagePath string - WorkDir string - PrefetchDir string + NydusImagePath string + WorkDir string + PrefetchPatterns string MultiPlatform bool DockerV2Format bool @@ -102,9 +102,9 @@ type Converter struct { CacheMaxRecords uint CacheVersion string - NydusImagePath string - WorkDir string - PrefetchDir string + NydusImagePath string + WorkDir string + PrefetchPatterns string MultiPlatform bool DockerV2Format bool @@ -146,7 +146,7 @@ func New(opt Opt) (*Converter, error) { CacheVersion: opt.CacheVersion, NydusImagePath: opt.NydusImagePath, WorkDir: opt.WorkDir, - PrefetchDir: opt.PrefetchDir, + PrefetchPatterns: opt.PrefetchPatterns, MultiPlatform: opt.MultiPlatform, DockerV2Format: opt.DockerV2Format, BackendForcePush: opt.BackendForcePush, @@ -207,10 +207,10 @@ func (cvt *Converter) convert(ctx context.Context) (retErr error) { return errors.Wrap(err, "Create bootstrap directory") } buildWorkflow, err := build.NewWorkflow(build.WorkflowOption{ - ChunkDict: chunkDictOpt, - NydusImagePath: cvt.NydusImagePath, - PrefetchDir: cvt.PrefetchDir, - TargetDir: cvt.WorkDir, + ChunkDict: chunkDictOpt, + NydusImagePath: cvt.NydusImagePath, + PrefetchPatterns: cvt.PrefetchPatterns, + TargetDir: cvt.WorkDir, }) if err != nil { return errors.Wrap(err, "Create build flow") diff --git a/contrib/nydusify/tests/nydusify.go b/contrib/nydusify/tests/nydusify.go index 26383517ba1..df67fe92a93 100644 --- a/contrib/nydusify/tests/nydusify.go +++ b/contrib/nydusify/tests/nydusify.go @@ -114,11 +114,11 @@ func (nydusify *Nydusify) Convert(t *testing.T) { CacheMaxRecords: 10, CacheVersion: "v1", - WorkDir: "./tmp", - PrefetchDir: "/", - NydusImagePath: nydusImagePath, - MultiPlatform: false, - DockerV2Format: true, + WorkDir: "./tmp", + PrefetchPatterns: "/", + NydusImagePath: nydusImagePath, + MultiPlatform: false, + DockerV2Format: true, BackendType: nydusify.backendType, BackendConfig: nydusify.backendConfig,