Merge pull request #306 from hhhhsdxxxx/henry.hj/nydusify

nydusify support pack subcommand

contrib/nydusify/cmd/nydusify.go

@@ -26,6 +26,7 @@ import (
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/converter/provider"
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/metrics"
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/metrics/fileexporter"
+	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/packer"
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/remote"
 	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils"
 )
@@ -336,6 +337,132 @@ func main() {
 				return checker.Check(context.Background())
 			},
 		},
+		{
+			Name:  "pack",
+			Usage: "Pack a directory to nydus bootstrap and blob",
+			Flags: []cli.Flag{
+				&cli.StringFlag{
+					Name:    "log-level",
+					Aliases: []string{"l"},
+					Value:   "info",
+					Usage:   "Set log level (panic, fatal, error, warn, info, debug, trace)",
+					EnvVars: []string{"LOG_LEVEL"},
+				},
+				&cli.StringFlag{
+					Name:     "source-dir",
+					Aliases:  []string{"target-dir"}, // for compatibility
+					Required: true,
+					Usage:    "The source directory of build target",
+					EnvVars:  []string{"SOURCE_DIR"},
+				},
+				&cli.StringFlag{
+					Name:     "output-dir",
+					Aliases:  []string{"o"},
+					Required: false,
+					Usage:    "Output dir of build artifact",
+					EnvVars:  []string{"OUTPUT_DIR"},
+				},
+				&cli.BoolFlag{
+					Name:    "backend-push",
+					Value:   false,
+					Usage:   "Push Nydus blob to storage backend",
+					EnvVars: []string{"BACKEND_PUSH"},
+				},
+				&cli.StringFlag{
+					Name:        "backend-type",
+					Value:       "oss",
+					DefaultText: "oss",
+					Usage:       "Specify Nydus blob storage backend type",
+					EnvVars:     []string{"BACKEND_TYPE"},
+				},
+				&cli.StringFlag{
+					Name:     "bootstrap",
+					Aliases:  []string{"meta"}, // for compatibility
+					Required: true,
+					Usage:    "Specify Nydus meta file name",
+					EnvVars:  []string{"BOOTSTRAP"},
+				},
+				&cli.StringFlag{
+					Name:    "parent-bootstrap",
+					Usage:   "Specify parent bootstrap to pack dictionary",
+					EnvVars: []string{"PARENT_BOOTSTRAP"},
+				},
+				&cli.StringFlag{
+					Name: "chunk-dict",
+					Usage: "Specify a chunk dict expression for chunk deduplication, " +
+						"for example: bootstrap=/path/to/dict.boot",
+					EnvVars: []string{"CHUNK_DICT"},
+				},
+				&cli.StringFlag{
+					Name:      "backend-config-file",
+					TakesFile: true,
+					Usage:     "Specify Nydus blob storage backend config from path",
+					EnvVars:   []string{"BACKEND_CONFIG_FILE"},
+				},
+				&cli.StringFlag{
+					Name:    "nydus-image",
+					Value:   "nydus-image",
+					Usage:   "The nydus-image binary path, if unset, search in PATH environment",
+					EnvVars: []string{"NYDUS_IMAGE"},
+				},
+			},
+			Before: func(ctx *cli.Context) error {
+				targetPath := ctx.String("target-dir")
+				fi, err := os.Stat(targetPath)
+				if err != nil {
+					return errors.Wrapf(err, "failed to stat target path %s", targetPath)
+				}
+				if !fi.IsDir() {
+					return errors.Errorf("%s is not a directory", targetPath)
+				}
+				return nil
+			},
+			Action: func(c *cli.Context) error {
+				var (
+					p             *packer.Packer
+					res           packer.PackResult
+					backendConfig *packer.BackendConfig
+					err           error
+				)
+
+				// if backend-push is specified, we should make sure backend-config-file exists
+				if c.Bool("backend-push") {
+					backendConfigFile := c.String("backend-config-file")
+					if strings.TrimSpace(backendConfigFile) == "" {
+						return errors.New("backend-config-file is required when specify --backend-push")
+					}
+					if _, err = os.Stat(backendConfigFile); err != nil {
+						return errors.Errorf("can not find backend-config-file %s", backendConfigFile)
+					}
+					cfg, err := packer.ParseBackendConfig(backendConfigFile)
+					if err != nil {
+						return errors.Errorf("failed to parse backend-config-file %s, err = %v", backendConfigFile, err)
+					}
+					backendConfig = &cfg
+				}
+
+				if p, err = packer.New(packer.Opt{
+					LogLevel:       c.String("log-level"),
+					NydusImagePath: c.String("nydus-image"),
+					OutputDir:      c.String("output-dir"),
+					BackendConfig:  backendConfig,
+				}); err != nil {
+					return err
+				}
+
+				if res, err = p.Pack(context.Background(), packer.PackRequest{
+					Parent:    c.String("parent-bootstrap"),
+					ChunkDict: c.String("chunk-dict"),
+					TargetDir: c.String("target-dir"),
+					Meta:      c.String("bootstrap"),
+					PushBlob:  c.Bool("backend-push"),
+				}); err != nil {
+					return err
+				}
+				logrus.Infof("successfully pack meta %s, blob %s", res.Meta, res.Blob)
+				return nil
+			},
+		},
 	}
 
 	// Under platform linux/arm64, containerd/compression prioritizes using `unpigz`

contrib/nydusify/go.sum

@@ -193,6 +193,7 @@ github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKv
 github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=

contrib/nydusify/pkg/packer/artifact.go

@@ -0,0 +1,40 @@
+package packer
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils"
+)
+
+type Artifact struct {
+	OutputDir string
+}
+
+func NewArtifact(outputDir string) (Artifact, error) {
+	res := Artifact{OutputDir: outputDir}
+	if err := res.ensureOutputDir(); err != nil {
+		return Artifact{}, err
+	}
+	return res, nil
+}
+
+func (a Artifact) bootstrapPath(metaFileName string) string {
+	return filepath.Join(a.OutputDir, metaFileName)
+}
+
+func (a Artifact) outputJSONPath() string {
+	return filepath.Join(a.OutputDir, "output.json")
+}
+
+func (a Artifact) blobFilePath(blobFileName string) string {
+	return filepath.Join(a.OutputDir, blobFileName)
+}
+
+// ensureOutputDir use user defined outputDir or defaultOutputDir, and make sure dir exists
+func (a *Artifact) ensureOutputDir() error {
+	if utils.IsEmptyString(a.OutputDir) {
+		a.OutputDir = defaultOutputDir
+	}
+	return os.MkdirAll(a.OutputDir, 0755)
+}