diff --git a/contrib/nydusify/cmd/nydusify.go b/contrib/nydusify/cmd/nydusify.go index 95c137b408e..c05966d371d 100644 --- a/contrib/nydusify/cmd/nydusify.go +++ b/contrib/nydusify/cmd/nydusify.go @@ -405,6 +405,18 @@ func main() { Usage: "The nydus-image binary path, if unset, search in PATH environment", EnvVars: []string{"NYDUS_IMAGE"}, }, + &cli.BoolFlag{ + Name: "compact", + Usage: "Compact parent bootstrap if necessary before do pack", + EnvVars: []string{"COMPACT"}, + }, + &cli.StringFlag{ + Name: "compact-config-file", + Usage: "Compact config file, default config is " + + "{\"min_used_ratio\": 5, \"compact_blob_size\": 10485760, \"max_compact_size\": 104857600, " + + "\"layers_to_compact\": 32}", + EnvVars: []string{"COMPACT_CONFIG_FILE"}, + }, }, Before: func(ctx *cli.Context) error { targetPath := ctx.String("target-dir") @@ -456,6 +468,9 @@ func main() { TargetDir: c.String("target-dir"), Meta: c.String("bootstrap"), PushBlob: c.Bool("backend-push"), + + TryCompact: c.Bool("compact"), + CompactConfigPath: c.String("compact-config-file"), }); err != nil { return err } diff --git a/contrib/nydusify/pkg/build/builder.go b/contrib/nydusify/pkg/build/builder.go index 769392d783b..959414425b6 100644 --- a/contrib/nydusify/pkg/build/builder.go +++ b/contrib/nydusify/pkg/build/builder.go @@ -28,6 +28,16 @@ type BuilderOption struct { AlignedChunk bool } +type CompactOption struct { + ChunkDict string + BootstrapPath string + OutputBootstrapPath string + BackendType string + BackendConfigPath string + OutputJSONPath string + CompactConfigPath string +} + type Builder struct { binaryPath string stdout io.Writer @@ -42,6 +52,50 @@ func NewBuilder(binaryPath string) *Builder { } } +func (builder *Builder) run(args []string, stdinInfo ...string) error { + logrus.Debugf("\tCommand: %s %s", builder.binaryPath, strings.Join(args[:], " ")) + + cmd := exec.Command(builder.binaryPath, args...) + cmd.Stdout = builder.stdout + cmd.Stderr = builder.stderr + + stdin, err := cmd.StdinPipe() + if err != nil { + return err + } + + for _, s := range stdinInfo { + io.WriteString(stdin, s) + } + stdin.Close() + + if err := cmd.Run(); err != nil { + logrus.WithError(err).Errorf("fail to run %v %+v", builder.binaryPath, args) + return err + } + + return nil +} + +func (builder *Builder) Compact(option CompactOption) error { + args := []string{ + "compact", + "--bootstrap", option.BootstrapPath, + "--config", option.CompactConfigPath, + "--backend-type", option.BackendType, + "--backend-config-file", option.BackendConfigPath, + "--log-level", "info", + "--output-json", option.OutputJSONPath, + } + if option.OutputBootstrapPath != "" { + args = append(args, "--output-bootstrap", option.OutputBootstrapPath) + } + if option.ChunkDict != "" { + args = append(args, "--chunk-dict", option.ChunkDict) + } + return builder.run(args) +} + // Run exec nydus-image CLI to build layer func (builder *Builder) Run(option BuilderOption) error { var args []string @@ -82,24 +136,5 @@ func (builder *Builder) Run(option BuilderOption) error { args = append(args, "--prefetch-policy", "fs") } - logrus.Debugf("\tCommand: %s %s", builder.binaryPath, strings.Join(args[:], " ")) - - cmd := exec.Command(builder.binaryPath, args...) - cmd.Stdout = builder.stdout - cmd.Stderr = builder.stderr - - stdin, err := cmd.StdinPipe() - if err != nil { - return err - } - - io.WriteString(stdin, option.PrefetchDir) - stdin.Close() - - if err := cmd.Run(); err != nil { - logrus.WithError(err).Errorf("fail to run %v %+v", builder.binaryPath, args) - return err - } - - return nil + return builder.run(args, option.PrefetchDir) } diff --git a/contrib/nydusify/pkg/compactor/compactor.go b/contrib/nydusify/pkg/compactor/compactor.go new file mode 100644 index 00000000000..dfda2bb9f89 --- /dev/null +++ b/contrib/nydusify/pkg/compactor/compactor.go @@ -0,0 +1,107 @@ +package compactor + +import ( + "encoding/json" + "os" + "path/filepath" + + "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/build" + "github.com/pkg/errors" +) + +var defaultCompactConfig = &CompactConfig{ + MinUsedRatio: 5, + CompactBlobSize: 10485760, + MaxCompactSize: 104857600, + LayersToCompact: 32, +} + +type CompactConfig struct { + MinUsedRatio int `json:"min_used_ratio"` + CompactBlobSize int `json:"compact_blob_size"` + MaxCompactSize int `json:"max_compact_size"` + LayersToCompact int `json:"layers_to_compact"` + BlobsDir string `json:"blobs_dir,omitempty"` +} + +func (cfg *CompactConfig) Dumps(filePath string) error { + file, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + return errors.Wrap(err, "open file failed") + } + defer file.Close() + if err = json.NewEncoder(file).Encode(cfg); err != nil { + return errors.Wrap(err, "encode json failed") + } + return nil +} + +func loadCompactConfig(filePath string) (CompactConfig, error) { + file, err := os.Open(filePath) + if err != nil { + return CompactConfig{}, errors.Wrap(err, "load compact config file failed") + } + defer file.Close() + var cfg CompactConfig + if err = json.NewDecoder(file).Decode(&cfg); err != nil { + return CompactConfig{}, errors.Wrap(err, "decode compact config file failed") + } + return cfg, nil +} + +type Compactor struct { + builder *build.Builder + workdir string + cfg CompactConfig +} + +func NewCompactor(nydusImagePath, workdir, configPath string) (*Compactor, error) { + var ( + cfg CompactConfig + err error + ) + if configPath != "" { + cfg, err = loadCompactConfig(configPath) + if err != nil { + return nil, errors.Wrap(err, "compact config err") + } + } else { + cfg = *defaultCompactConfig + } + cfg.BlobsDir = workdir + return &Compactor{ + builder: build.NewBuilder(nydusImagePath), + workdir: workdir, + cfg: cfg, + }, nil +} + +func (compactor *Compactor) Compact(bootstrapPath, chunkDict, backendType, backendConfigFile string) (string, error) { + targetBootstrap := bootstrapPath + ".compact" + if err := os.Remove(targetBootstrap); err != nil && !os.IsNotExist(err) { + return "", errors.Wrap(err, "delete old target bootstrap failed") + } + // prepare config file + configFilePath := filepath.Join(compactor.workdir, "compact.json") + if err := compactor.cfg.Dumps(configFilePath); err != nil { + return "", errors.Wrap(err, "compact err") + } + outputJSONPath := filepath.Join(compactor.workdir, "compact-result.json") + if err := os.Remove(outputJSONPath); err != nil && !os.IsNotExist(err) { + return "", errors.Wrap(err, "delete old output-json file failed") + } + err := compactor.builder.Compact(build.CompactOption{ + ChunkDict: chunkDict, + BootstrapPath: bootstrapPath, + OutputBootstrapPath: targetBootstrap, + BackendType: backendType, + BackendConfigPath: backendConfigFile, + OutputJSONPath: outputJSONPath, + CompactConfigPath: configFilePath, + }) + if err != nil { + return "", errors.Wrap(err, "run compact command failed") + } + + return targetBootstrap, nil +} diff --git a/contrib/nydusify/pkg/packer/packer.go b/contrib/nydusify/pkg/packer/packer.go index c116af05c40..208887b7694 100644 --- a/contrib/nydusify/pkg/packer/packer.go +++ b/contrib/nydusify/pkg/packer/packer.go @@ -12,7 +12,9 @@ import ( "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/build" "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/checker/tool" + "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/compactor" "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -93,6 +95,9 @@ type PackRequest struct { ChunkDict string // PushBlob whether to push blob and meta to remote backend PushBlob bool + + TryCompact bool + CompactConfigPath string } type PackResult struct { @@ -198,16 +203,78 @@ func (p *Packer) getNewBlobsHash(exists []string) (string, error) { return "", nil } +func (p *Packer) dumpBlobBackendConfig(filePath string) (func(), error) { + file, err := os.OpenFile(filePath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return nil, err + } + defer file.Close() + n, err := file.Write(p.BackendConfig.rawBlobBackendCfg()) + if err != nil { + return nil, err + } + return func() { + zeros := make([]byte, n) + file, err = os.OpenFile(filePath, os.O_WRONLY, 0644) + if err != nil { + logrus.Errorf("open config file %s failed err = %v", filePath, err) + return + } + file.Write(zeros) + file.Close() + os.Remove(filePath) + }, nil +} + +func (p *Packer) tryCompactParent(req *PackRequest) error { + if !req.TryCompact || req.Parent == "" || p.BackendConfig == nil { + return nil + } + // dumps backend config file + backendConfigPath := filepath.Join(p.OutputDir, "backend-config.json") + destroy, err := p.dumpBlobBackendConfig(backendConfigPath) + if err != nil { + return errors.Wrap(err, "dump backend config file failed") + } + // destroy backend config file, because there are secrets + defer destroy() + c, err := compactor.NewCompactor(p.nydusImagePath, p.OutputDir, req.CompactConfigPath) + if err != nil { + return errors.Wrap(err, "new compactor failed") + } + // only support oss now + outputBootstrap, err := c.Compact(req.Parent, req.ChunkDict, "oss", backendConfigPath) + if err != nil { + return errors.Wrap(err, "compact parent failed") + } + // check output bootstrap + _, err = os.Stat(outputBootstrap) + if err != nil && !os.IsNotExist(err) { + return errors.Wrapf(err, "stat target bootstrap failed") + } + if err == nil { + // parent --> output bootstrap + p.logger.Infof("compact bootstrap %s successfully, use parent %s", req.Parent, outputBootstrap) + req.Parent = outputBootstrap + } + + return nil +} + func (p *Packer) Pack(_ context.Context, req PackRequest) (PackResult, error) { p.logger.Infof("start to pack source directory %q", req.TargetDir) + if err := p.tryCompactParent(&req); err != nil { + p.logger.Errorf("try compact parent bootstrap err %v", err) + return PackResult{}, err + } blobPath := p.blobFilePath(blobFileName(req.Meta)) parentBlobs, err := p.getBlobsFromBootstrap(req.Parent) if err != nil { - return PackResult{}, err + return PackResult{}, errors.Wrap(err, "get blobs from parent bootstrap failed") } chunkDictBlobs, err := p.getChunkDictBlobs(req.ChunkDict) if err != nil { - return PackResult{}, err + return PackResult{}, errors.Wrap(err, "get blobs from chunk-dict failed") } if err = p.builder.Run(build.BuilderOption{ ParentBootstrapPath: req.Parent, @@ -226,10 +293,9 @@ func (p *Packer) Pack(_ context.Context, req PackRequest) (PackResult, error) { } if newBlobHash == "" { blobPath = "" - } - if req.Parent != "" { - p.logger.Infof("should make sure parent blobs already exists on oss or local") - if newBlobHash != "" { + } else { + if req.Parent != "" || req.PushBlob { + p.logger.Infof("rename blob file into sha256 csum") if err = os.Rename(blobPath, p.blobFilePath(newBlobHash)); err != nil { return PackResult{}, errors.Wrap(err, "failed to rename blob file") } @@ -251,6 +317,8 @@ func (p *Packer) Pack(_ context.Context, req PackRequest) (PackResult, error) { pushResult, err := p.pusher.Push(PushRequest{ Meta: req.Meta, Blob: newBlobHash, + + ParentBlobs: parentBlobs, }) if err != nil { return PackResult{}, errors.Wrap(err, "failed to push pack result to remote") diff --git a/contrib/nydusify/pkg/packer/pusher.go b/contrib/nydusify/pkg/packer/pusher.go index 65aff215fdf..72d49af7482 100644 --- a/contrib/nydusify/pkg/packer/pusher.go +++ b/contrib/nydusify/pkg/packer/pusher.go @@ -25,6 +25,8 @@ type Pusher struct { type PushRequest struct { Meta string Blob string + + ParentBlobs []string } type PushResult struct { @@ -69,10 +71,18 @@ func NewPusher(opt NewPusherOpt) (*Pusher, error) { // and blob file name is the hash of the blobfile that is extracted from output.json func (p *Pusher) Push(req PushRequest) (PushResult, error) { p.logger.Info("start to push meta and blob to remote backend") - p.logger.Infof("push blob %s", req.Blob) // todo: add a suitable timeout ctx := context.Background() // todo: use blob desc to build manifest + + for _, blob := range req.ParentBlobs { + // try push parent blobs + if _, err := p.blobBackend.Upload(ctx, blob, p.blobFilePath(blob), 0, false); err != nil { + return PushResult{}, errors.Wrap(err, "failed to put blobfile to remote") + } + } + + p.logger.Infof("push blob %s", req.Blob) if req.Blob != "" { if _, err := p.blobBackend.Upload(ctx, req.Blob, p.blobFilePath(req.Blob), 0, false); err != nil { return PushResult{}, errors.Wrap(err, "failed to put blobfile to remote")