Merge pull request #344 from imeoer/song/nydusify-prefetch-patterns

Nydusify: pipe stdin to builder if enable --prefetch-patterns

contrib/nydusify/cmd/nydusify.go

@@ -112,6 +112,35 @@ func getCacheReference(c *cli.Context, target string) (string, error) {
 	return cache, nil
 }
 
+func getPrefetchPatterns(c *cli.Context) (string, error) {
+	prefetchedDir := c.String("prefetch-dir")
+	prefetchPatterns := c.Bool("prefetch-patterns")
+
+	if len(prefetchedDir) > 0 && prefetchPatterns {
+		return "", fmt.Errorf("--prefetch-dir conflicts with --prefetch-patterns")
+	}
+
+	var patterns string
+
+	if prefetchPatterns {
+		bytes, err := ioutil.ReadAll(os.Stdin)
+		if err != nil {
+			return "", errors.Wrap(err, "read prefetch patterns from STDIN")
+		}
+		patterns = string(bytes)
+	}
+
+	if len(prefetchedDir) > 0 {
+		patterns = prefetchedDir
+	}
+
+	if len(patterns) <= 0 {
+		patterns = "/"
+	}
+
+	return patterns, nil
+}
+
 func main() {
 	logrus.SetFormatter(&logrus.TextFormatter{
 		FullTimestamp: true,
@@ -139,7 +168,8 @@ func main() {
 				&cli.BoolFlag{Name: "source-insecure", Required: false, Usage: "Allow http/insecure source registry communication", EnvVars: []string{"SOURCE_INSECURE"}},
 				&cli.BoolFlag{Name: "target-insecure", Required: false, Usage: "Allow http/insecure target registry communication", EnvVars: []string{"TARGET_INSECURE"}},
 				&cli.StringFlag{Name: "work-dir", Value: "./tmp", Usage: "Work directory path for image conversion", EnvVars: []string{"WORK_DIR"}},
-				&cli.StringFlag{Name: "prefetch-dir", Value: "/", Usage: "Prefetch directory for nydus image, use absolute path of rootfs", EnvVars: []string{"PREFETCH_DIR"}},
+				&cli.StringFlag{Name: "prefetch-dir", Value: "", Usage: "Prefetched directory for nydus image, use absolute path of rootfs", EnvVars: []string{"PREFETCH_DIR"}},
+				&cli.BoolFlag{Name: "prefetch-patterns", Value: false, Usage: "Prefetched file path patterns from STDIN, specify absolute/relative path of rootfs line by line", EnvVars: []string{"PREFETCH_PATTERNS"}},
 				&cli.StringFlag{Name: "nydus-image", Value: "nydus-image", Usage: "The nydus-image binary path, if unset, search in PATH environment", EnvVars: []string{"NYDUS_IMAGE"}},
 				&cli.BoolFlag{Name: "multi-platform", Value: false, Usage: "Merge OCI & Nydus manifest to manifest index for target image, please ensure that OCI manifest already exists in target image", EnvVars: []string{"MULTI_PLATFORM"}},
 				&cli.StringFlag{Name: "platform", Value: "linux/" + runtime.GOARCH, Usage: "Let nydusify choose image of specified platform from manifest index. Possible value is `amd64` or `arm64`"},
@@ -238,6 +268,11 @@ func main() {
 					return err
 				}
 
+				prefetchPatterns, err := getPrefetchPatterns(c)
+				if err != nil {
+					return err
+				}
+
 				opt := converter.Opt{
 					Logger:          logger,
 					SourceProviders: sourceProviders,
@@ -248,11 +283,11 @@ func main() {
 					CacheMaxRecords: cacheMaxRecords,
 					CacheVersion:    cacheVersion,
 
-					WorkDir:        c.String("work-dir"),
-					PrefetchDir:    c.String("prefetch-dir"),
-					NydusImagePath: c.String("nydus-image"),
-					MultiPlatform:  c.Bool("multi-platform"),
-					DockerV2Format: c.Bool("docker-v2-format"),
+					WorkDir:          c.String("work-dir"),
+					PrefetchPatterns: prefetchPatterns,
+					NydusImagePath:   c.String("nydus-image"),
+					MultiPlatform:    c.Bool("multi-platform"),
+					DockerV2Format:   c.Bool("docker-v2-format"),
 
 					BackendType:         backendType,
 					BackendConfig:       backendConfig,

contrib/nydusify/examples/converter/main.go

@@ -50,11 +50,11 @@ func main() {
 		SourceProviders: sourceProviders,
 		TargetRemote:    targetRemote,
 
-		WorkDir:        wordDir,
-		PrefetchDir:    "/",
-		NydusImagePath: nydusImagePath,
-		MultiPlatform:  false,
-		DockerV2Format: true,
+		WorkDir:          wordDir,
+		PrefetchPatterns: "/",
+		NydusImagePath:   nydusImagePath,
+		MultiPlatform:    false,
+		DockerV2Format:   true,
 	}
 
 	cvt, err := converter.New(opt)

contrib/nydusify/pkg/build/builder.go

@@ -20,9 +20,9 @@ type BuilderOption struct {
 	RootfsPath          string
 	BackendType         string
 	BackendConfig       string
-	PrefetchDir         string
 	WhiteoutSpec        string
 	OutputJSONPath      string
+	PrefetchPatterns    string
 	// A regular file or fifo into which commands nydus-image to dump contents.
 	BlobPath     string
 	AlignedChunk bool
@@ -52,22 +52,13 @@ func NewBuilder(binaryPath string) *Builder {
 	}
 }
 
-func (builder *Builder) run(args []string, stdinInfo ...string) error {
+func (builder *Builder) run(args []string, prefetchPatterns string) error {
 	logrus.Debugf("\tCommand: %s %s", builder.binaryPath, strings.Join(args[:], " "))
 
 	cmd := exec.Command(builder.binaryPath, args...)
 	cmd.Stdout = builder.stdout
 	cmd.Stderr = builder.stderr
-
-	stdin, err := cmd.StdinPipe()
-	if err != nil {
-		return err
-	}
-
-	for _, s := range stdinInfo {
-		io.WriteString(stdin, s)
-	}
-	stdin.Close()
+	cmd.Stdin = strings.NewReader(prefetchPatterns)
 
 	if err := cmd.Run(); err != nil {
 		logrus.WithError(err).Errorf("fail to run %v %+v", builder.binaryPath, args)
@@ -93,7 +84,7 @@ func (builder *Builder) Compact(option CompactOption) error {
 	if option.ChunkDict != "" {
 		args = append(args, "--chunk-dict", option.ChunkDict)
 	}
-	return builder.run(args)
+	return builder.run(args, "")
 }
 
 // Run exec nydus-image CLI to build layer
@@ -132,9 +123,9 @@ func (builder *Builder) Run(option BuilderOption) error {
 		option.RootfsPath,
 	)
 
-	if option.PrefetchDir != "" {
+	if len(option.PrefetchPatterns) > 0 {
 		args = append(args, "--prefetch-policy", "fs")
 	}
 
-	return builder.run(args, option.PrefetchDir)
+	return builder.run(args, option.PrefetchPatterns)
 }

contrib/nydusify/pkg/build/workflow.go

@@ -17,10 +17,10 @@ import (
 )
 
 type WorkflowOption struct {
-	ChunkDict      string
-	TargetDir      string
-	NydusImagePath string
-	PrefetchDir    string
+	ChunkDict        string
+	TargetDir        string
+	NydusImagePath   string
+	PrefetchPatterns string
 }
 
 type Workflow struct {
@@ -88,10 +88,6 @@ func NewWorkflow(option WorkflowOption) (*Workflow, error) {
 	backendConfig := fmt.Sprintf(`{"dir": "%s"}`, blobsDir)
 	builder := NewBuilder(option.NydusImagePath)
 
-	if option.PrefetchDir == "" {
-		option.PrefetchDir = "/"
-	}
-
 	return &Workflow{
 		WorkflowOption: option,
 		blobsDir:       blobsDir,
@@ -116,7 +112,7 @@ func (workflow *Workflow) Build(
 		ParentBootstrapPath: workflow.parentBootstrapPath,
 		BootstrapPath:       workflow.bootstrapPath,
 		RootfsPath:          layerDir,
-		PrefetchDir:         workflow.PrefetchDir,
+		PrefetchPatterns:    workflow.PrefetchPatterns,
 		WhiteoutSpec:        whiteoutSpec,
 		OutputJSONPath:      workflow.buildOutputJSONPath(),
 		BlobPath:            blobPath,

contrib/nydusify/pkg/converter/converter.go

@@ -74,9 +74,9 @@ type Opt struct {
 	CacheMaxRecords uint
 	CacheVersion    string
 
-	NydusImagePath string
-	WorkDir        string
-	PrefetchDir    string
+	NydusImagePath   string
+	WorkDir          string
+	PrefetchPatterns string
 
 	MultiPlatform  bool
 	DockerV2Format bool
@@ -102,9 +102,9 @@ type Converter struct {
 	CacheMaxRecords uint
 	CacheVersion    string
 
-	NydusImagePath string
-	WorkDir        string
-	PrefetchDir    string
+	NydusImagePath   string
+	WorkDir          string
+	PrefetchPatterns string
 
 	MultiPlatform  bool
 	DockerV2Format bool
@@ -146,7 +146,7 @@ func New(opt Opt) (*Converter, error) {
 		CacheVersion:        opt.CacheVersion,
 		NydusImagePath:      opt.NydusImagePath,
 		WorkDir:             opt.WorkDir,
-		PrefetchDir:         opt.PrefetchDir,
+		PrefetchPatterns:    opt.PrefetchPatterns,
 		MultiPlatform:       opt.MultiPlatform,
 		DockerV2Format:      opt.DockerV2Format,
 		BackendForcePush:    opt.BackendForcePush,
@@ -207,10 +207,10 @@ func (cvt *Converter) convert(ctx context.Context) (retErr error) {
 		return errors.Wrap(err, "Create bootstrap directory")
 	}
 	buildWorkflow, err := build.NewWorkflow(build.WorkflowOption{
-		ChunkDict:      chunkDictOpt,
-		NydusImagePath: cvt.NydusImagePath,
-		PrefetchDir:    cvt.PrefetchDir,
-		TargetDir:      cvt.WorkDir,
+		ChunkDict:        chunkDictOpt,
+		NydusImagePath:   cvt.NydusImagePath,
+		PrefetchPatterns: cvt.PrefetchPatterns,
+		TargetDir:        cvt.WorkDir,
 	})
 	if err != nil {
 		return errors.Wrap(err, "Create build flow")

contrib/nydusify/tests/nydusify.go

@@ -114,11 +114,11 @@ func (nydusify *Nydusify) Convert(t *testing.T) {
 		CacheMaxRecords: 10,
 		CacheVersion:    "v1",
 
-		WorkDir:        "./tmp",
-		PrefetchDir:    "/",
-		NydusImagePath: nydusImagePath,
-		MultiPlatform:  false,
-		DockerV2Format: true,
+		WorkDir:          "./tmp",
+		PrefetchPatterns: "/",
+		NydusImagePath:   nydusImagePath,
+		MultiPlatform:    false,
+		DockerV2Format:   true,
 
 		BackendType:   nydusify.backendType,
 		BackendConfig: nydusify.backendConfig,