From bb507ac930fffc4b5490f79bb582c9c83b2fb384 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 4 Jan 2025 00:15:20 +0100 Subject: [PATCH 01/73] chore(lint): refacto setParam to remove complexity --- pkg/cli/util/util.go | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/pkg/cli/util/util.go b/pkg/cli/util/util.go index a8cc28c0c..58c4cbaa1 100644 --- a/pkg/cli/util/util.go +++ b/pkg/cli/util/util.go @@ -33,15 +33,24 @@ type LDFlags struct { Date string } -func SetParam(c *cli.Context, logE *logrus.Entry, commandName string, param *config.Param, ldFlags *LDFlags) error { //nolint:funlen,cyclop +func SetParam(c *cli.Context, logE *logrus.Entry, commandName string, param *config.Param, ldFlags *LDFlags) error { wd, err := os.Getwd() if err != nil { return fmt.Errorf("get the current directory: %w", err) } param.Args = c.Args().Slice() - if logLevel := c.String("log-level"); logLevel != "" { - param.LogLevel = logLevel + setBasicParams(c, logE, commandName, param, wd, ldFlags) + setLogParams(c, param, logE) + if err := setEnvParams(param); err != nil { + return fmt.Errorf("error during setting params from Env vars: %w", err) + } + if err := setChecksumParams(param); err != nil { + return fmt.Errorf("error during setting params from checksum params: %w", err) } + return nil +} + +func setBasicParams(c *cli.Context, logE *logrus.Entry, commandName string, param *config.Param, wd string, ldFlags *LDFlags) { param.ConfigFilePath = c.String("config") param.Dest = c.String("o") param.OutTestData = c.String("out-testdata") @@ -66,14 +75,11 @@ func SetParam(c *cli.Context, logE *logrus.Entry, commandName string, param *con if cmd := c.String("cmd"); cmd != "" { param.Commands = strings.Split(cmd, ",") } - param.LogColor = os.Getenv("AQUA_LOG_COLOR") param.AQUAVersion = ldFlags.Version param.AquaCommitHash = ldFlags.Commit param.RootDir = config.GetRootDir(osenv.New()) homeDir, _ := os.UserHomeDir() param.HomeDir = homeDir - log.SetLevel(param.LogLevel, logE) - log.SetColor(param.LogColor, logE) param.MaxParallelism = config.GetMaxParallelism(os.Getenv("AQUA_MAX_PARALLELISM"), logE) param.GlobalConfigFilePaths = finder.ParseGlobalConfigFilePaths(wd, os.Getenv("AQUA_GLOBAL_CONFIG")) param.Deep = c.Bool("deep") @@ -84,7 +90,18 @@ func SetParam(c *cli.Context, logE *logrus.Entry, commandName string, param *con param.ProgressBar = os.Getenv("AQUA_PROGRESS_BAR") == "true" param.Tags = parseTags(strings.Split(c.String("tags"), ",")) param.ExcludedTags = parseTags(strings.Split(c.String("exclude-tags"), ",")) +} +func setLogParams(c *cli.Context, param *config.Param, logE *logrus.Entry) { + if logLevel := c.String("log-level"); logLevel != "" { + param.LogLevel = logLevel + } + param.LogColor = os.Getenv("AQUA_LOG_COLOR") + log.SetLevel(param.LogLevel, logE) + log.SetColor(param.LogColor, logE) +} + +func setEnvParams(param *config.Param) error { if a := os.Getenv("AQUA_DISABLE_LAZY_INSTALL"); a != "" { disableLazyInstall, err := strconv.ParseBool(a) if err != nil { @@ -108,6 +125,10 @@ func SetParam(c *cli.Context, logE *logrus.Entry, commandName string, param *con } } } + return nil +} + +func setChecksumParams(param *config.Param) error { if a := os.Getenv("AQUA_CHECKSUM"); a != "" { chksm, err := strconv.ParseBool(a) if err != nil { From cf7cc90493dfbd18d4fe8d5fcfc5d9a499fa1b80 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Wed, 8 Jan 2025 00:05:55 +0100 Subject: [PATCH 02/73] chore(deps): add go-humanize and bbolt dependencies --- go.mod | 2 ++ go.sum | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/go.mod b/go.mod index d11dc5624..b07b4c5c1 100644 --- a/go.mod +++ b/go.mod @@ -48,6 +48,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect + github.com/dustin/go-humanize v1.0.1 github.com/gdamore/encoding v1.0.0 // indirect github.com/gdamore/tcell/v2 v2.6.0 // indirect github.com/google/go-querystring v1.1.0 // indirect @@ -82,6 +83,7 @@ require ( github.com/therootcompany/xz v1.0.1 // indirect github.com/ulikunitz/xz v0.5.12 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect + go.etcd.io/bbolt v1.3.11 go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/crypto v0.28.0 // indirect golang.org/x/sync v0.9.0 // indirect diff --git a/go.sum b/go.sum index 50927ddbb..229a826b8 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/expr-lang/expr v1.16.9 h1:WUAzmR0JNI9JCiF0/ewwHB1gmcGw5wW7nWt8gc6PpCI= @@ -255,6 +257,8 @@ github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBi github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= +go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= From c867e8d5bff2d293cc815163626e71be366ea80e Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Wed, 8 Jan 2025 00:07:04 +0100 Subject: [PATCH 03/73] feat(vacuum): implement StoreQueue and Controller for vacuum management --- pkg/controller/vacuum/controller.go | 169 ++++++++ pkg/controller/vacuum/queue_store.go | 86 ++++ pkg/controller/vacuum/vacuum.go | 613 +++++++++++++++++++++++++++ 3 files changed, 868 insertions(+) create mode 100644 pkg/controller/vacuum/controller.go create mode 100644 pkg/controller/vacuum/queue_store.go create mode 100644 pkg/controller/vacuum/vacuum.go diff --git a/pkg/controller/vacuum/controller.go b/pkg/controller/vacuum/controller.go new file mode 100644 index 000000000..b1c4cbee4 --- /dev/null +++ b/pkg/controller/vacuum/controller.go @@ -0,0 +1,169 @@ +package vacuum + +import ( + "fmt" + "io" + "os" + "sync" + "sync/atomic" + "time" + + "github.com/aquaproj/aqua/v2/pkg/config" + "github.com/sirupsen/logrus" + "github.com/spf13/afero" + bolt "go.etcd.io/bbolt" +) + +type Controller struct { + stdout io.Writer + dbMutex sync.RWMutex + db atomic.Pointer[bolt.DB] + Param *config.Param + fs afero.Fs + storeQueue *StoreQueue +} + +// New initializes a Controller with the given context, parameters, and dependencies. +func New(param *config.Param, fs afero.Fs) *Controller { + vc := &Controller{ + stdout: os.Stdout, + Param: param, + fs: fs, + } + vc.storeQueue = NewStoreQueue(vc) + return vc +} + +// getDB retrieves the database instance, initializing it if necessary. +func (vc *Controller) getDB() (*bolt.DB, error) { + if db := vc.db.Load(); db != nil { + return db, nil + } + + vc.dbMutex.Lock() + defer vc.dbMutex.Unlock() + + if db := vc.db.Load(); db != nil { + return db, nil + } + + const dbFileMode = 0o600 + db, err := bolt.Open(vc.Param.RootDir+"/"+dbFile, dbFileMode, &bolt.Options{ + Timeout: 1 * time.Second, + }) + if err != nil { + return nil, fmt.Errorf("failed to open database %v: %w", dbFile, err) + } + + if err := db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte(bucketNamePkgs)) + if err != nil { + return fmt.Errorf("failed to create bucket '%v' in '%v': %w", bucketNamePkgs, dbFile, err) + } + return nil + }); err != nil { + db.Close() + return nil, fmt.Errorf("failed to create bucket: %w", err) + } + + vc.db.Store(db) + return db, nil +} + +// withDBRetry retries a database operation with exponential backoff. +func (vc *Controller) withDBRetry(logE *logrus.Entry, fn func(*bolt.Tx) error, dbAccessType DBAccessType) error { + const ( + retries = 2 + initialBackoff = 100 * time.Millisecond + exponentialBackoff = 2 + ) + backoff := initialBackoff + for i := range retries { + err := vc.withDB(logE, fn, dbAccessType) + if err == nil { + return nil + } + + logE.WithFields(logrus.Fields{ + "attempt": i + 1, + "error": err, + }).Warn("Retrying database operation") + + time.Sleep(backoff) + backoff *= exponentialBackoff + } + + return fmt.Errorf("database operation failed after %d retries", retries) +} + +// withDB executes a function within a database transaction. +func (vc *Controller) withDB(logE *logrus.Entry, fn func(*bolt.Tx) error, dbAccessType DBAccessType) error { + db, err := vc.getDB() + if err != nil { + return err + } + if db == nil { + return nil + } + defer func() { + if err := vc.closeDB(); err != nil { + logE.WithError(err).Error("Failed to close database") + } + }() + + if dbAccessType == Update { + err = db.Update(fn) + if err != nil { + return fmt.Errorf("failed to view database: %w", err) + } + return nil + } + err = db.View(fn) + if err != nil { + return fmt.Errorf("failed to view database: %w", err) + } + return nil +} + +// Keep_DBOpen opens the database instance. This is used for testing purposes. +func (vc *Controller) TestKeepDBOpen() error { + const dbFileMode = 0o600 + _, _ = bolt.Open(vc.Param.RootDir+"/"+dbFile, dbFileMode, &bolt.Options{ + Timeout: 1 * time.Second, + }) + return nil +} + +// closeDB closes the database instance. +func (vc *Controller) closeDB() error { + vc.dbMutex.Lock() + defer vc.dbMutex.Unlock() + + if vc.db.Load() != nil { + if err := vc.db.Load().Close(); err != nil { + return fmt.Errorf("failed to close database: %w", err) + } + vc.db.Store(nil) + } + + return nil +} + +// Close closes the dependencies of the Controller. +func (vc *Controller) close(logE *logrus.Entry) error { + logE.Debug("Closing vacuum controller") + if vc.storeQueue != nil { + vc.storeQueue.close() + } + + vc.dbMutex.Lock() + defer vc.dbMutex.Unlock() + + if vc.db.Load() != nil { + if err := vc.db.Load().Close(); err != nil { + return fmt.Errorf("failed to close database: %w", err) + } + vc.db.Store(nil) + } + return nil +} diff --git a/pkg/controller/vacuum/queue_store.go b/pkg/controller/vacuum/queue_store.go new file mode 100644 index 000000000..8d1aae246 --- /dev/null +++ b/pkg/controller/vacuum/queue_store.go @@ -0,0 +1,86 @@ +package vacuum + +import ( + "sync" + + "github.com/sirupsen/logrus" +) + +// storeRequest represents a task to be processed. +type StoreRequest struct { + logE *logrus.Entry + pkg []*ConfigPackage +} + +// StoreQueue manages a queue for handling tasks sequentially. +type StoreQueue struct { + taskQueue chan StoreRequest + wg sync.WaitGroup + vc *Controller + done chan struct{} + closeOnce sync.Once +} + +// NewStoreQueue initializes the task queue with a single worker. +func NewStoreQueue(vc *Controller) *StoreQueue { + const maxTasks = 100 + sq := &StoreQueue{ + taskQueue: make(chan StoreRequest, maxTasks), + done: make(chan struct{}), + vc: vc, + } + + go sq.worker() + return sq +} + +// worker processes tasks from the queue. +func (sq *StoreQueue) worker() { + for { + select { + case task, ok := <-sq.taskQueue: + if !ok { + return + } + err := sq.vc.storePackageInternal(task.logE, task.pkg) + if err != nil { + task.logE.WithField("vacuum", dbFile).WithError(err).Error("Failed to store package asynchronously") + } + sq.wg.Done() + case <-sq.done: + // Process remaining tasks + for len(sq.taskQueue) > 0 { + task := <-sq.taskQueue + err := sq.vc.storePackageInternal(task.logE, task.pkg) + if err != nil { + task.logE.WithError(err).Error("Failed to store package asynchronously during shutdown") + } + sq.wg.Done() + } + return + } + } +} + +// Enqueue adds a task to the queue. +func (sq *StoreQueue) enqueue(logE *logrus.Entry, pkg []*ConfigPackage) { + select { + case <-sq.done: + return + default: + sq.wg.Add(1) + sq.taskQueue <- StoreRequest{ + logE: logE, + pkg: pkg, + } + } +} + +// Close waits for all tasks to complete and stops the worker. +func (sq *StoreQueue) close() { + sq.closeOnce.Do(func() { + close(sq.done) + sq.wg.Wait() + close(sq.taskQueue) + }) +} diff --git a/pkg/controller/vacuum/vacuum.go b/pkg/controller/vacuum/vacuum.go new file mode 100644 index 000000000..6e88490d5 --- /dev/null +++ b/pkg/controller/vacuum/vacuum.go @@ -0,0 +1,613 @@ +package vacuum + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "regexp" + "strings" + "sync" + "time" + + "github.com/aquaproj/aqua/v2/pkg/config" + "github.com/dustin/go-humanize" + "github.com/ktr0731/go-fuzzyfinder" + "github.com/sirupsen/logrus" + "github.com/spf13/afero" + "go.etcd.io/bbolt" +) + +type DBAccessType string + +const ( + dbFile string = "vacuum.db" + bucketNamePkgs string = "packages" + View DBAccessType = "view" + Update DBAccessType = "update" +) + +type PackageVacuumEntries []PackageVacuumEntry + +type PackageVacuumEntry struct { + Key []byte + PackageEntry *PackageEntry +} + +type ConfigPackage struct { + Type string + Name string + Version string + PkgPath []string +} + +type PackageEntry struct { + LastUsageTime time.Time + PkgPath []string +} + +type Mode string + +const ( + ListPackages Mode = "list-packages" + ListExpiredPackages Mode = "list-expired-packages" + StorePackage Mode = "store-package" + StorePackages Mode = "store-packages" + AsyncStorePackage Mode = "async-store-package" + VacuumExpiredPackages Mode = "vacuum-expired-packages" + Close Mode = "close" +) + +// GetVacuumModeCLI returns the corresponding Mode based on the provided mode string. +// It supports the following modes: +// - "list-packages": returns ListPackages mode +// - "list-expired-packages": returns ListExpiredPackages mode +// - "vacuum-expired-packages": returns VacuumExpiredPackages mode +// If the provided mode string does not match any of the supported modes, it returns an error. +// +// Parameters: +// +// mode (string): The mode string to be converted to a Mode. +// +// Returns: +// +// (Mode, error): The corresponding Mode and nil if the mode string is valid, otherwise an empty Mode and an error. +func (vc *Controller) GetVacuumModeCLI(mode string) (Mode, error) { + switch mode { + case "list-packages": + return ListPackages, nil + case "list-expired-packages": + return ListExpiredPackages, nil + case "vacuum-expired-packages": + return VacuumExpiredPackages, nil + default: + return "", errors.New("invalid vacuum mode") + } +} + +// Vacuum performs various vacuum operations based on the provided mode. +// Main function of vacuum controller. +func (vc *Controller) Vacuum(_ context.Context, logE *logrus.Entry, mode Mode, configPkg []*config.Package, args ...string) error { + if !vc.IsVacuumEnabled(logE) { + return nil + } + + vacuumPkg := convertConfigPackages(configPkg) + + switch mode { + case ListPackages: + return vc.handleListPackages(logE, args...) + case ListExpiredPackages: + return vc.handleListExpiredPackages(logE, args...) + case StorePackage: + return vc.handleStorePackage(logE, vacuumPkg) + case StorePackages: + return vc.handleStorePackages(logE, vacuumPkg) + case AsyncStorePackage: + return vc.handleAsyncStorePackage(logE, vacuumPkg) + case VacuumExpiredPackages: + return vc.handleVacuumExpiredPackages(logE) + case Close: + return vc.close(logE) + } + + return errors.New("invalid vacuum mode") +} + +// convertConfigPackages converts a slice of config.Package pointers to a slice of ConfigPackage pointers. +func convertConfigPackages(configPkg []*config.Package) []*ConfigPackage { + vacuumPkg := make([]*ConfigPackage, 0, len(configPkg)) + for _, pkg := range configPkg { + vacuumPkg = append(vacuumPkg, vacuumConfigPackageFromConfigPackage(pkg)) + } + return vacuumPkg +} + +// handleListPackages retrieves a list of packages and displays them using a fuzzy search. +func (vc *Controller) handleListPackages(logE *logrus.Entry, args ...string) error { + pkgs, err := vc.listPackages(logE) + if err != nil { + return err + } + return vc.displayPackagesFuzzy(logE, pkgs, args...) +} + +// handleListExpiredPackages handles the process of listing expired packages +// and displaying them using a fuzzy search. +func (vc *Controller) handleListExpiredPackages(logE *logrus.Entry, args ...string) error { + expiredPkgs, err := vc.listExpiredPackages(logE) + if err != nil { + return err + } + return vc.displayPackagesFuzzy(logE, expiredPkgs, args...) +} + +// handleStorePackage processes a list of configuration packages and stores the first package in the list. +func (vc *Controller) handleStorePackage(logE *logrus.Entry, vacuumPkg []*ConfigPackage) error { + if len(vacuumPkg) < 1 { + return errors.New("StorePackage requires at least one configPackage") + } + defer func() { + if err := vc.close(logE); err != nil { + logE.WithError(err).Error("Failed to close vacuum DB after storing package") + } + }() + return vc.storePackageInternal(logE, []*ConfigPackage{vacuumPkg[0]}) +} + +// handleStorePackages processes a list of configuration packages and stores them. +func (vc *Controller) handleStorePackages(logE *logrus.Entry, vacuumPkg []*ConfigPackage) error { + if len(vacuumPkg) < 1 { + return errors.New("StorePackages requires at least one configPackage") + } + defer func() { + if err := vc.close(logE); err != nil { + logE.WithError(err).Error("Failed to close vacuum DB after storing multiple packages") + } + }() + return vc.storePackageInternal(logE, vacuumPkg) +} + +// handleAsyncStorePackage processes a list of configuration packages asynchronously. +func (vc *Controller) handleAsyncStorePackage(logE *logrus.Entry, vacuumPkg []*ConfigPackage) error { + if len(vacuumPkg) < 1 { + return errors.New("AsyncStorePackage requires at least one configPackage") + } + vc.storeQueue.enqueue(logE, vacuumPkg) + return nil +} + +// handleVacuumExpiredPackages handles the process of vacuuming expired packages. +func (vc *Controller) handleVacuumExpiredPackages(logE *logrus.Entry) error { + return vc.vacuumExpiredPackages(logE) +} + +// IsVacuumEnabled checks if the vacuum feature is enabled based on the configuration. +func (vc *Controller) IsVacuumEnabled(logE *logrus.Entry) bool { + if vc.Param.VacuumDays == nil || *vc.Param.VacuumDays <= 0 { + logE.Debug("Vacuum is disabled. AQUA_VACUUM_DAYS is not set or invalid.") + return false + } + return true +} + +// listExpiredPackages lists all packages that have expired based on the vacuum configuration. +func (vc *Controller) listExpiredPackages(logE *logrus.Entry) ([]*PackageVacuumEntry, error) { + pkgs, err := vc.listPackages(logE) + if err != nil { + return nil, err + } + + var expired []*PackageVacuumEntry + for _, pkg := range pkgs { + if vc.isPackageExpired(pkg) { + expired = append(expired, pkg) + } + } + return expired, nil +} + +// isPackageExpired checks if a package is expired based on the vacuum configuration. +func (vc *Controller) isPackageExpired(pkg *PackageVacuumEntry) bool { + const secondsInADay = 24 * 60 * 60 + threshold := int64(*vc.Param.VacuumDays) * secondsInADay + return time.Since(pkg.PackageEntry.LastUsageTime).Seconds() > float64(threshold) +} + +// listPackages lists all stored package entries. +func (vc *Controller) listPackages(logE *logrus.Entry) ([]*PackageVacuumEntry, error) { + db, err := vc.getDB() + if err != nil { + return nil, err + } + if db == nil { + return nil, nil + } + defer db.Close() + + var pkgs []*PackageVacuumEntry + + err = vc.withDBRetry(logE, func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(bucketNamePkgs)) + if b == nil { + return nil + } + return b.ForEach(func(k, value []byte) error { + pkgEntry, err := decodePackageEntry(value) + if err != nil { + logE.WithFields(logrus.Fields{ + "pkgKey": string(k), + }).Warnf("Failed to decode entry: %v", err) + return err + } + pkgs = append(pkgs, &PackageVacuumEntry{ + Key: append([]byte{}, k...), + PackageEntry: pkgEntry, + }) + return nil + }) + }, View) + return pkgs, err +} + +func (vc *Controller) displayPackagesFuzzyTest(logE *logrus.Entry, pkgs []*PackageVacuumEntry) error { + var pkgInformations struct { + TotalPackages int + TotalExpired int + } + for _, pkg := range pkgs { + if vc.isPackageExpired(pkg) { + pkgInformations.TotalExpired++ + } + pkgInformations.TotalPackages++ + } + // Display log entry with informations for testing purposes + logE.WithFields(logrus.Fields{ + "TotalPackages": pkgInformations.TotalPackages, + "TotalExpired": pkgInformations.TotalExpired, + }).Info("Test mode: Displaying packages") + return nil +} + +func (vc *Controller) displayPackagesFuzzy(logE *logrus.Entry, pkgs []*PackageVacuumEntry, args ...string) error { + if len(pkgs) == 0 { + logE.Info("No packages to display") + return nil + } + if len(args) > 0 && args[0] == "test" { + return vc.displayPackagesFuzzyTest(logE, pkgs) + } + return vc.displayPackagesFuzzyInteractive(pkgs) +} + +func (vc *Controller) displayPackagesFuzzyInteractive(pkgs []*PackageVacuumEntry) error { + _, err := fuzzyfinder.Find(pkgs, func(i int) string { + var expiredString string + if vc.isPackageExpired(pkgs[i]) { + expiredString = "⌛ " + } + + return fmt.Sprintf("%s%s [%s]", + expiredString, + pkgs[i].Key, + humanize.Time(pkgs[i].PackageEntry.LastUsageTime), + ) + }, + fuzzyfinder.WithPreviewWindow(func(i, _, _ int) string { + if i == -1 { + return "No package selected" + } + pkg := pkgs[i] + var expiredString string + if vc.isPackageExpired(pkg) { + expiredString = "Expired ⌛" + } + parsedConfigPkg, err := generateConfigPackageFromKey(pkg.Key) + if err != nil { + return fmt.Sprintf("Failed to parse package key: %v", err) + } + return fmt.Sprintf( + "Package Details:\n\n"+ + "%s \n"+ + "Type: %s\n"+ + "Package: %s\n"+ + "Version: %s\n\n"+ + "Last Used: %s\n"+ + "Last Used (exact): %s\n\n"+ + "PkgPath: %v\n", + expiredString, + parsedConfigPkg.Type, + parsedConfigPkg.Name, + parsedConfigPkg.Version, + humanize.Time(pkg.PackageEntry.LastUsageTime), + pkg.PackageEntry.LastUsageTime.Format("2024-12-31 15:04:05"), + pkg.PackageEntry.PkgPath, + ) + }), + fuzzyfinder.WithHeader("Navigate through packages to display details"), + ) + if err != nil { + if errors.Is(err, fuzzyfinder.ErrAbort) { + return nil + } + return fmt.Errorf("failed to display packages: %w", err) + } + + return nil +} + +// vacuumExpiredPackages performs cleanup of expired packages. +func (vc *Controller) vacuumExpiredPackages(logE *logrus.Entry) error { + expired, err := vc.listExpiredPackages(logE) + if err != nil { + return err + } + + if len(expired) == 0 { + return nil + } + + successKeys, errCh := vc.processExpiredPackages(logE, expired) + + if len(errCh) > 0 { + return errors.New("some packages could not be removed") + } + if len(successKeys) > 0 { + if err := vc.removePackages(logE, successKeys); err != nil { + return fmt.Errorf("failed to remove packages from database: %w", err) + } + } + + return nil +} + +// processExpiredPackages processes a list of expired package entries by removing their associated paths +// and generating a list of configuration packages to be removed from vacuum database. +// +// Parameters: +// - logE: A logrus.Entry used for logging errors and information. +// - expired: A slice of PackageVacuumEntry representing the expired packages to be processed. +// +// Returns: +// - A slice of ConfigPackage representing the packages that were successfully processed and need to be removed from the vacuum database. +// - A slice of errors encountered during the processing of the expired packages. +func (vc *Controller) processExpiredPackages(logE *logrus.Entry, expired []*PackageVacuumEntry) ([]*ConfigPackage, []error) { //nolint:funlen + const batchSize = 10 + successKeys := make(chan string, len(expired)) + errCh := make(chan error, len(expired)) + + var wg sync.WaitGroup + for i := 0; i < len(expired); i += batchSize { + end := i + batchSize + if end > len(expired) { + end = len(expired) + } + + batch := make([]struct { + key string + pkgPath []string + version string + }, len(expired[i:end])) + + for j, entry := range expired[i:end] { + batch[j].key = string(entry.Key) + batch[j].pkgPath = entry.PackageEntry.PkgPath + batch[j].version = strings.Split(string(entry.Key), "@")[1] + } + + wg.Add(1) + go func(batch []struct { + key string + pkgPath []string + version string + }, + ) { + defer wg.Done() + for _, entry := range batch { + for _, path := range entry.pkgPath { + if err := vc.removePackageVersionPath(vc.Param, path, entry.version); err != nil { + logE.WithField("expiredPackages", entry.key).WithError(err).Error("Error removing path") + errCh <- err + continue + } + successKeys <- entry.key + } + } + }(batch) + } + + wg.Wait() + close(successKeys) + close(errCh) + + ConfigPackageToRemove := make([]*ConfigPackage, 0, len(expired)) + for key := range successKeys { + pkg, err := generateConfigPackageFromKey([]byte(key)) + if err != nil { + logE.WithField("key", key).WithError(err).Error("Failed to generate package from key") + continue + } + ConfigPackageToRemove = append(ConfigPackageToRemove, pkg) + } + + errors := make([]error, 0, len(expired)) + for err := range errCh { + errors = append(errors, err) + } + + return ConfigPackageToRemove, errors +} + +// storePackageInternal stores package entries in the database. +func (vc *Controller) storePackageInternal(logE *logrus.Entry, pkgs []*ConfigPackage, dateTime ...time.Time) error { + lastUsedTime := time.Now() + if len(dateTime) > 0 { + lastUsedTime = dateTime[0] + } + return vc.withDBRetry(logE, func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(bucketNamePkgs)) + if b == nil { + return errors.New("bucket not found") + } + + for _, pkg := range pkgs { + logE.WithFields(logrus.Fields{ + "name": pkg.Name, + "version": pkg.Version, + "PkgPath": pkg.PkgPath, + }).Debug("Storing package in vacuum database") + pkgEntry := &PackageEntry{ + LastUsageTime: lastUsedTime, + PkgPath: pkg.PkgPath, + } + + data, err := encodePackageEntry(pkgEntry) + if err != nil { + logE.WithFields(logrus.Fields{ + "name": pkg.Name, + "version": pkg.Version, + }).WithError(err).Error("Failed to encode package") + return fmt.Errorf("encode package %s: %w", pkg.Name, err) + } + + if err := b.Put(generateKey(pkg), data); err != nil { + logE.WithField("pkgKey", pkg).WithError(err).Error("Failed to store package in vacuum database") + return fmt.Errorf("store package %s: %w", pkg.Name, err) + } + } + return nil + }, Update) +} + +// removePackages removes package entries from the database. +func (vc *Controller) removePackages(logE *logrus.Entry, pkgs []*ConfigPackage) error { + keys := make([]string, 0, len(pkgs)) + for _, pkg := range pkgs { + keys = append(keys, string(generateKey(pkg))) + } + return vc.withDBRetry(logE, func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(bucketNamePkgs)) + if b == nil { + return errors.New("bucket not found") + } + + for _, key := range keys { + if err := b.Delete([]byte(key)); err != nil { + return fmt.Errorf("delete package %s: %w", key, err) + } + } + return nil + }, Update) +} + +// removePackageVersionPath removes the specified package version directory and its parent directory if it becomes empty. +func (vc *Controller) removePackageVersionPath(param *config.Param, path string, version string) error { + pkgVersionPath := filepath.Join(param.RootDir, "pkgs", path, version) + if err := vc.fs.RemoveAll(pkgVersionPath); err != nil { + return fmt.Errorf("remove package version directories: %w", err) + } + + pkgPath := filepath.Join(param.RootDir, "pkgs", path) + dirIsEmpty, err := afero.IsEmpty(vc.fs, pkgPath) + if err != nil { + return fmt.Errorf("check if the directory is empty: %w", err) + } + if dirIsEmpty { + if err := vc.fs.RemoveAll(pkgPath); err != nil { + return fmt.Errorf("remove package directories: %w", err) + } + } + return nil +} + +// encodePackageEntry encodes a PackageEntry into a JSON byte slice. +func encodePackageEntry(pkgEntry *PackageEntry) ([]byte, error) { + data, err := json.Marshal(pkgEntry) + if err != nil { + return nil, fmt.Errorf("failed to marshal package entry: %w", err) + } + return data, nil +} + +// decodePackageEntry decodes a JSON byte slice into a PackageEntry. +func decodePackageEntry(data []byte) (*PackageEntry, error) { + var pkgEntry PackageEntry + if err := json.Unmarshal(data, &pkgEntry); err != nil { + return nil, fmt.Errorf("failed to unmarshal package entry: %w", err) + } + return &pkgEntry, nil +} + +// retrievePackageEntry retrieves a package entry from the database by key. +func (vc *Controller) retrievePackageEntry(logE *logrus.Entry, key []byte) (*PackageEntry, error) { + var pkgEntry *PackageEntry + err := vc.withDBRetry(logE, func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(bucketNamePkgs)) + if b == nil { + return nil + } + value := b.Get(key) + if value == nil { + return nil + } + + var err error + pkgEntry, err = decodePackageEntry(value) + return err + }, View) + return pkgEntry, err +} + +// generateKey generates a unique key for a package. +func generateKey(pkg *ConfigPackage) []byte { + return []byte(pkg.Type + "," + pkg.Name + "@" + pkg.Version) +} + +// vacuumConfigPackageFromConfigPackage returns a ConfigPackage config from a config.Package. +func vacuumConfigPackageFromConfigPackage(pkg *config.Package) *ConfigPackage { + pkgPathsMap := pkg.PackageInfo.PkgPaths() + PkgPath := make([]string, 0, len(pkgPathsMap)) + for k := range pkgPathsMap { + PkgPath = append(PkgPath, k) + } + + return &ConfigPackage{ + Type: pkg.PackageInfo.Type, + Name: pkg.Package.Name, + Version: pkg.Package.Version, + PkgPath: PkgPath, + } +} + +// generateConfigPackageFromKey return a minimal package config from a key. +func generateConfigPackageFromKey(key []byte) (*ConfigPackage, error) { + if len(key) == 0 { + return nil, errors.New("empty key") + } + pattern := `^(?P[^,]+),(?P[^@]+)@(?P.+)$` + re := regexp.MustCompile(pattern) + match := re.FindStringSubmatch(string(key)) + + if match == nil { + return nil, fmt.Errorf("key %s does not match the pattern %s", key, pattern) + } + + result := make(map[string]string) + for i, name := range re.SubexpNames() { + if i != 0 && name != "" { + result[name] = match[i] + } + } + + packageInfoType := result["PackageInfo_Type"] + packageName := result["Package_Name"] + packageVersion := result["Package_Version"] + + pkg := &ConfigPackage{ + Type: packageInfoType, + Name: packageName, + Version: packageVersion, + } + return pkg, nil +} From 90afe12df5f5eb7b91c6f07d2272b14d052531cf Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Wed, 8 Jan 2025 00:07:40 +0100 Subject: [PATCH 04/73] test(vacuum): add unit tests for vacuum controller --- pkg/controller/vacuum/vacuum_internal_test.go | 78 +++ pkg/controller/vacuum/vacuum_test.go | 585 ++++++++++++++++++ 2 files changed, 663 insertions(+) create mode 100644 pkg/controller/vacuum/vacuum_internal_test.go create mode 100644 pkg/controller/vacuum/vacuum_test.go diff --git a/pkg/controller/vacuum/vacuum_internal_test.go b/pkg/controller/vacuum/vacuum_internal_test.go new file mode 100644 index 000000000..c1fb460ce --- /dev/null +++ b/pkg/controller/vacuum/vacuum_internal_test.go @@ -0,0 +1,78 @@ +package vacuum + +import ( + "testing" + "time" + + "github.com/aquaproj/aqua/v2/pkg/config" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGenerateConfigPackageFromKey(t *testing.T) { + t.Parallel() + testCases := []struct { + name string + key string + expected *ConfigPackage + expectError bool + }{ + { + name: "Valid key", + key: "github_release,test/pkg@v1.0.0", + expected: &ConfigPackage{ + Type: "github_release", + Name: "test/pkg", + Version: "v1.0.0", + }, + expectError: false, + }, + { + name: "Invalid key format", + key: "invalid_key_format", + expected: nil, + expectError: true, + }, + { + name: "Empty key", + key: "", + expected: nil, + expectError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + result, err := generateConfigPackageFromKey([]byte(tc.key)) + if tc.expectError { + require.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, tc.expected, result) + } + }) + } +} + +// GetPackageLastUsed retrieves the last used time of a package. for testing purposes. +func (vc *Controller) GetPackageLastUsed(logE *logrus.Entry, pkg *config.Package) *time.Time { + var lastUsedTime time.Time + vacuumpkg := vacuumConfigPackageFromConfigPackage(pkg) + key := generateKey(vacuumpkg) + pkgEntry, _ := vc.retrievePackageEntry(logE, key) + if pkgEntry != nil { + lastUsedTime = pkgEntry.LastUsageTime + } + return &lastUsedTime +} + +// SetTimeStampPackage permit define a Timestamp for a package Manually. for testing purposes. +func (vc *Controller) SetTimestampPackages(logE *logrus.Entry, pkg []*config.Package, datetime time.Time) error { + vacuumPkgs := make([]*ConfigPackage, 0, len(pkg)) + for _, p := range pkg { + vacuumPkgs = append(vacuumPkgs, vacuumConfigPackageFromConfigPackage(p)) + } + return vc.storePackageInternal(logE, vacuumPkgs, datetime) +} diff --git a/pkg/controller/vacuum/vacuum_test.go b/pkg/controller/vacuum/vacuum_test.go new file mode 100644 index 000000000..823490ce6 --- /dev/null +++ b/pkg/controller/vacuum/vacuum_test.go @@ -0,0 +1,585 @@ +package vacuum_test + +import ( + "context" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/aquaproj/aqua/v2/pkg/config" + "github.com/aquaproj/aqua/v2/pkg/config/aqua" + "github.com/aquaproj/aqua/v2/pkg/config/registry" + "github.com/aquaproj/aqua/v2/pkg/controller/vacuum" + "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus/hooks/test" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + ListPackages string = "list-packages" + ListExpiredPackages string = "list-expired-packages" + StorePackage string = "store-package" + StorePackages string = "store-packages" + AsyncStorePackage string = "async-store-package" + VacuumExpiredPackages string = "vacuum-expired-packages" + Close string = "close" +) + +func TestVacuum(t *testing.T) { //nolint:funlen,maintidx + t.Parallel() + // Setup common test fixtures + logger, hook := test.NewNullLogger() + logE := logrus.NewEntry(logger) + + ctx := context.Background() + fs := afero.NewOsFs() + + // Create temp directory for tests + tempTestDir, err := afero.TempDir(fs, "/tmp", "vacuum_test") + require.NoError(t, err) + t.Cleanup(func() { + err := fs.RemoveAll(tempTestDir) + if err != nil { + t.Fatal(err) + } + }) + + t.Run("vacuum disabled", func(t *testing.T) { + t.Parallel() + testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_disabled") + require.NoError(t, err) + // Setup + param := &config.Param{ + VacuumDays: nil, + RootDir: testDir, + } + controller := vacuum.New(param, fs) + + // Test + err = controller.Vacuum(ctx, logE, vacuum.Mode(ListPackages), nil) + + // Assert + require.NoError(t, err, "Should return nil when vacuum is disabled") + }) + + t.Run("invalid mode", func(t *testing.T) { + t.Parallel() + testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_invalid_mode") + require.NoError(t, err) + // Setup + days := 30 + param := &config.Param{ + VacuumDays: &days, + RootDir: testDir, + } + controller := vacuum.New(param, fs) + + // Test + err = controller.Vacuum(ctx, logE, vacuum.Mode("invalid"), nil) + + // Assert + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid vacuum mode") + }) + + t.Run("ListPackages mode - empty database", func(t *testing.T) { + t.Parallel() + // Setup - use a new temp directory for this test + testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_list_test") + require.NoError(t, err) + + days := 30 + param := &config.Param{ + VacuumDays: &days, + RootDir: testDir, + } + controller := vacuum.New(param, fs) + + // Test + err = controller.Vacuum(ctx, logE, vacuum.Mode(ListPackages), nil, "test") + + // Assert + require.NoError(t, err) // Should succeed with empty database + assert.Equal(t, "No packages to display", hook.LastEntry().Message) + }) + + t.Run("AsyncFailed", func(t *testing.T) { + t.Parallel() + testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_async_failed") + require.NoError(t, err) + + days := 1 // Short expiration for testing + param := &config.Param{ + VacuumDays: &days, + RootDir: testDir, + } + controller := vacuum.New(param, fs) + + numberPackagesToStore := 4 + pkgs := generateTestPackages(numberPackagesToStore) + + // We force Keeping the DB open to simulate a failure in the async operation + if err := controller.TestKeepDBOpen(); err != nil { + t.Fatal(err) + } + hook.Reset() + for _, pkg := range pkgs { + err := controller.Vacuum(context.Background(), logE, vacuum.Mode(AsyncStorePackage), []*config.Package{pkg}) + require.NoError(t, err) + } + + // Wait for the async operations to complete + err = controller.Vacuum(context.Background(), logE, vacuum.Mode(Close), nil) + require.NoError(t, err) // If AsyncStorePackage fails, Close should wait for the async operations to complete, but not return an error + + expectedLogMessage := []string{ + "Failed to store package asynchronously", + "Retrying database operation", + "Failed to store package asynchronously during shutdown", + } + var receivedMessages []string + for _, entry := range hook.AllEntries() { + receivedMessages = append(receivedMessages, entry.Message) + } + for _, entry := range expectedLogMessage { + assert.Contains(t, receivedMessages, entry) + } + }) + + t.Run("StorePackage and ListPackages workflow", func(t *testing.T) { + t.Parallel() + // Setup - use a new temp directory for this test + testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_store_test") + require.NoError(t, err) + + days := 30 + param := &config.Param{ + VacuumDays: &days, + RootDir: testDir, + } + controller := vacuum.New(param, fs) + + numberPackagesToStore := 1 + pkgs := generateTestPackages(numberPackagesToStore) + + // Store the package + err = controller.Vacuum(ctx, logE, vacuum.Mode(StorePackage), pkgs) + require.NoError(t, err) + + // List packages - should contain our stored package + err = controller.Vacuum(ctx, logE, vacuum.Mode(ListPackages), nil, "test") + require.NoError(t, err) + assert.Equal(t, "Test mode: Displaying packages", hook.LastEntry().Message) + assert.Equal(t, 1, hook.LastEntry().Data["TotalPackages"]) + assert.Equal(t, 0, hook.LastEntry().Data["TotalExpired"]) + hook.Reset() + + // Verify package was stored correctly + lastUsed := controller.GetPackageLastUsed(logE, pkgs[0]) + assert.False(t, lastUsed.IsZero(), "Package should have a last used time") + }) + + t.Run("StorePackages", func(t *testing.T) { + t.Parallel() + testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_storePackages_test") + require.NoError(t, err) + + days := 30 + param := &config.Param{ + VacuumDays: &days, + RootDir: testDir, + } + controller := vacuum.New(param, fs) + + numberPackagesToStore := 4 + pkgs := generateTestPackages(numberPackagesToStore) + + // Store the package + err = controller.Vacuum(ctx, logE, vacuum.Mode(StorePackages), pkgs) + require.NoError(t, err) + + // List packages - should contain our stored package + err = controller.Vacuum(ctx, logE, vacuum.Mode(ListPackages), nil, "test") + require.NoError(t, err) + assert.Equal(t, "Test mode: Displaying packages", hook.LastEntry().Message) + assert.Equal(t, 4, hook.LastEntry().Data["TotalPackages"]) + assert.Equal(t, 0, hook.LastEntry().Data["TotalExpired"]) + hook.Reset() + }) + + t.Run("GetVacuumModeCLI valid modes", func(t *testing.T) { + t.Parallel() + testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_getVacuumModeCLI_test") + require.NoError(t, err) + param := &config.Param{ + RootDir: testDir, + } + controller := vacuum.New(param, fs) + + // Test valid modes + modes := map[string]vacuum.Mode{ + "list-packages": vacuum.ListPackages, + "list-expired-packages": vacuum.ListExpiredPackages, + "vacuum-expired-packages": vacuum.VacuumExpiredPackages, + } + + for modeStr, expectedMode := range modes { + mode, err := controller.GetVacuumModeCLI(modeStr) + require.NoError(t, err) + assert.Equal(t, expectedMode, mode) + } + }) + + t.Run("GetVacuumModeCLI invalid mode", func(t *testing.T) { + t.Parallel() + testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_getVacuumModeCLI_invalid_test") + require.NoError(t, err) + param := &config.Param{ + RootDir: testDir, + } + controller := vacuum.New(param, fs) + + // Test invalid mode + mode, err := controller.GetVacuumModeCLI("invalid-mode") + require.Error(t, err) + assert.Equal(t, vacuum.Mode(""), mode) + assert.Contains(t, err.Error(), "invalid vacuum mode") + }) + + t.Run("handleListExpiredPackages - no expired packages", func(t *testing.T) { + t.Parallel() + testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_handle_list_expired") + require.NoError(t, err) + + days := 30 + param := &config.Param{ + VacuumDays: &days, + RootDir: testDir, + } + controller := vacuum.New(param, fs) + + // Test + err = controller.Vacuum(ctx, logE, vacuum.Mode(ListExpiredPackages), nil, "test") + + // Assert + require.NoError(t, err) // Error if no package found + assert.Equal(t, "No packages to display", hook.LastEntry().Message) + }) + + t.Run("handleStorePackage - invalid package count", func(t *testing.T) { + t.Parallel() + testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_handle_store_package") + require.NoError(t, err) + days := 30 + param := &config.Param{ + VacuumDays: &days, + RootDir: testDir, + } + controller := vacuum.New(param, fs) + + // Test with no packages + err = controller.Vacuum(ctx, logE, vacuum.Mode(StorePackage), nil) + + // Assert + require.Error(t, err) + assert.Contains(t, err.Error(), "StorePackage requires at least one configPackage") + }) + + t.Run("handleStorePackages - invalid package count", func(t *testing.T) { + t.Parallel() + testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_handle_store_packages") + require.NoError(t, err) + days := 30 + param := &config.Param{ + VacuumDays: &days, + RootDir: testDir, + } + controller := vacuum.New(param, fs) + + // Test with no packages + err = controller.Vacuum(ctx, logE, vacuum.Mode(StorePackages), nil) + + // Assert + require.Error(t, err) + assert.Contains(t, err.Error(), "StorePackages requires at least one configPackage") + }) + + t.Run("handleAsyncStorePackage - invalid package count", func(t *testing.T) { + t.Parallel() + testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_handle_async_store_package") + require.NoError(t, err) + days := 30 + param := &config.Param{ + VacuumDays: &days, + RootDir: testDir, + } + controller := vacuum.New(param, fs) + + // Test with no packages + err = controller.Vacuum(ctx, logE, vacuum.Mode(AsyncStorePackage), nil) + + // Assert + require.Error(t, err) + assert.Contains(t, err.Error(), "AsyncStorePackage requires at least one configPackage") + }) + + t.Run("VacuumExpiredPackages workflow", func(t *testing.T) { + t.Parallel() + testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_expire_test") + require.NoError(t, err) + defer func() { + hook.Reset() + fs.RemoveAll(testDir) //nolint:errcheck + }() + + days := 1 // Short expiration for testing + param := &config.Param{ + VacuumDays: &days, + RootDir: testDir, + } + controller := vacuum.New(param, fs) + + numberPackagesToStore := 3 + numberPackagesToExpire := 1 + pkgs := generateTestPackages(numberPackagesToStore) + pkgPaths := make([]string, 0, len(pkgs)) + + // Create package paths and files + for _, pkg := range pkgs { + pkgPath := filepath.Join(testDir, "pkgs", "github_release/github.com/"+pkg.Package.Name, pkg.Package.Version) + err = fs.MkdirAll(pkgPath, 0o755) + require.NoError(t, err) + + // Create a test file in the package directory + testFile := filepath.Join(pkgPath, "test.txt") + err = afero.WriteFile(fs, testFile, []byte("test content"), 0o644) + require.NoError(t, err) + + pkgPaths = append(pkgPaths, pkgPath) + } + + // Store Multiple packages + err = controller.Vacuum(ctx, logE, vacuum.Mode(AsyncStorePackage), pkgs) + require.NoError(t, err) + + // Call Close to ensure all async operations are completed + err = controller.Vacuum(ctx, logE, vacuum.Mode(Close), nil) + require.NoError(t, err) + + // Modify timestamp of one package to be expired + oldTime := time.Now().Add(-48 * time.Hour) // 2 days old + err = controller.SetTimestampPackages(logE, pkgs[0:numberPackagesToExpire], oldTime) + require.NoError(t, err) + + // Check Packages after expiration + err = controller.Vacuum(ctx, logE, vacuum.Mode(ListPackages), nil, "test") + require.NoError(t, err) + assert.Equal(t, numberPackagesToStore, hook.LastEntry().Data["TotalPackages"]) + assert.Equal(t, numberPackagesToExpire, hook.LastEntry().Data["TotalExpired"]) + + // List expired packages only + err = controller.Vacuum(ctx, logE, vacuum.Mode(ListExpiredPackages), nil, "test") + require.NoError(t, err) + assert.Equal(t, numberPackagesToExpire, hook.LastEntry().Data["TotalPackages"]) + assert.Equal(t, numberPackagesToExpire, hook.LastEntry().Data["TotalExpired"]) + + // Run vacuum + err = controller.Vacuum(ctx, logE, vacuum.Mode(VacuumExpiredPackages), nil) + require.NoError(t, err) + + // List expired packages + err = controller.Vacuum(ctx, logE, vacuum.Mode(ListPackages), nil, "test") + require.NoError(t, err) + assert.Equal(t, numberPackagesToStore-numberPackagesToExpire, hook.LastEntry().Data["TotalPackages"]) + assert.Equal(t, 0, hook.LastEntry().Data["TotalExpired"]) + + // Verify Package Paths was removed : + for _, pkgPath := range pkgPaths[:numberPackagesToExpire] { + exist, err := afero.Exists(fs, pkgPath) + require.NoError(t, err) + assert.False(t, exist, "Package directory should be removed after vacuum") + } + + // Modify timestamp of one package to be expired And lock DB to simulate a failure in the vacuum operation + err = controller.SetTimestampPackages(logE, pkgs[numberPackagesToExpire:], oldTime) + require.NoError(t, err) + + // Keep Database open to simulate a failure in the vacuum operation + err = controller.TestKeepDBOpen() + require.NoError(t, err) + + // Run vacuum + err = controller.Vacuum(ctx, logE, vacuum.Mode(VacuumExpiredPackages), nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to open database vacuum.db: timeout") + }) + + t.Run("TestVacuumWithoutExpiredPackages", func(t *testing.T) { + t.Parallel() + testDir, err := afero.TempDir(afero.NewOsFs(), "", "vacuum_no_expired") + require.NoError(t, err) + + days := 30 + param := &config.Param{ + VacuumDays: &days, + RootDir: testDir, + } + controller := vacuum.New(param, afero.NewOsFs()) + + // Store non-expired packages + pkgs := generateTestPackages(3) + err = controller.Vacuum(ctx, logE, vacuum.Mode(StorePackages), pkgs) + require.NoError(t, err) + + // Run vacuum + err = controller.Vacuum(ctx, logE, vacuum.Mode(VacuumExpiredPackages), nil) + require.NoError(t, err) + + // Verify no packages were removed + err = controller.Vacuum(ctx, logE, vacuum.Mode(ListPackages), nil, "test") + require.NoError(t, err) + assert.Equal(t, 3, hook.LastEntry().Data["TotalPackages"]) + }) +} + +func generateTestPackages(count int) []*config.Package { + pkgs := make([]*config.Package, count) + for i := range count { + pkgs[i] = &config.Package{ + Package: &aqua.Package{ + Name: "cli/cli", + Version: "v2." + string(rune(i+'0')) + ".0", + Registry: "standard", + }, + PackageInfo: ®istry.PackageInfo{ + Type: "github_release", + RepoOwner: "cli", + RepoName: "cli", + Asset: "gh_2." + string(rune(i+'0')) + ".0_linux_amd64.tar.gz", + }, + } + } + return pkgs +} + +// BenchmarkVacuum_StorePackages benchmarks the performance of storing packages. +func BenchmarkVacuum_StorePackages(b *testing.B) { + benchmarkVacuumStorePackages(b, 100) +} + +// BenchmarkVacuum_OnlyOneStorePackage benchmarks the performance of storing only one package. +func BenchmarkVacuum_OnlyOneStorePackage(b *testing.B) { + benchmarkVacuumStorePackages(b, 1) +} + +// benchmarkVacuumStorePackages is a helper function to benchmark the performance of storing packages. +func benchmarkVacuumStorePackages(b *testing.B, pkgCount int) { //nolint:cyclop,funlen,gocognit + b.Helper() + pkgs, logE, syncParam, syncMultipleParam, asyncParam, asyncMultipleParam, fs, syncf, syncMultiplef, asyncf, asyncMultiplef := setupBenchmark(b, pkgCount) + defer func() { + if err := fs.RemoveAll(syncf); err != nil { + b.Fatal(err) + } + if err := fs.RemoveAll(syncMultiplef); err != nil { + b.Fatal(err) + } + if err := fs.RemoveAll(asyncf); err != nil { + b.Fatal(err) + } + if err := fs.RemoveAll(asyncMultiplef); err != nil { + b.Fatal(err) + } + }() + + b.Run("Sync", func(b *testing.B) { + controller := vacuum.New(syncParam, fs) + b.ResetTimer() + for range b.N { + for _, pkg := range pkgs { + if err := controller.Vacuum(context.Background(), logE, vacuum.Mode(StorePackage), []*config.Package{pkg}); err != nil { + b.Fatal(err) + } + } + } + }) + + b.Run("SyncMultipleSameTime", func(b *testing.B) { + controller := vacuum.New(syncMultipleParam, fs) + b.ResetTimer() + for range b.N { + if err := controller.Vacuum(context.Background(), logE, vacuum.Mode(StorePackage), pkgs); err != nil { + b.Fatal(err) + } + } + }) + + b.Run("Async", func(b *testing.B) { + controller := vacuum.New(asyncParam, fs) + b.ResetTimer() + for range b.N { + runtime.MemProfileRate = 1 + for _, pkg := range pkgs { + if err := controller.Vacuum(context.Background(), logE, vacuum.Mode(AsyncStorePackage), []*config.Package{pkg}); err != nil { + b.Fatal(err) + } + } + if err := controller.Vacuum(context.Background(), logE, vacuum.Mode(Close), nil); err != nil { + b.Fatal(err) + } + } + }) + + b.Run("AsyncMultiple", func(b *testing.B) { + controller := vacuum.New(asyncMultipleParam, fs) + b.ResetTimer() + for range b.N { + if err := controller.Vacuum(context.Background(), logE, vacuum.Mode(AsyncStorePackage), pkgs); err != nil { + b.Fatal(err) + } + if err := controller.Vacuum(context.Background(), logE, vacuum.Mode(Close), nil); err != nil { + b.Fatal(err) + } + } + }) +} + +func setupBenchmark(b *testing.B, pkgCount int) ([]*config.Package, *logrus.Entry, *config.Param, *config.Param, *config.Param, *config.Param, afero.Fs, string, string, string, string) { + b.Helper() + pkgs := generateTestPackages(pkgCount) + vacuumDays := 5 + logE := logrus.NewEntry(logrus.New()) + fs := afero.NewOsFs() + + // Benchmark sync configuration + syncf, errf := afero.TempDir(fs, "/tmp", "vacuum_test_sync") + if errf != nil { + b.Fatal(errf) + } + syncParam := &config.Param{RootDir: syncf, VacuumDays: &vacuumDays} + + // Benchmark SyncMultipleSameTime configuration + syncMultiplef, errf := afero.TempDir(fs, "/tmp", "vacuum_test_multiple") + if errf != nil { + b.Fatal(errf) + } + syncMultipleParam := &config.Param{RootDir: syncMultiplef, VacuumDays: &vacuumDays} + + // Benchmark async configuration + asyncf, errf := afero.TempDir(fs, "/tmp", "vacuum_test_async") + if errf != nil { + b.Fatal(errf) + } + asyncParam := &config.Param{RootDir: asyncf, VacuumDays: &vacuumDays} + + // Benchmark async multiple configuration + asyncMultiplef, errf := afero.TempDir(fs, "/tmp", "vacuum_test_async_multiple") + if errf != nil { + b.Fatal(errf) + } + asyncMultipleParam := &config.Param{RootDir: asyncMultiplef, VacuumDays: &vacuumDays} + + return pkgs, logE, syncParam, syncMultipleParam, asyncParam, asyncMultipleParam, fs, syncf, syncMultiplef, asyncf, asyncMultiplef +} From 491a793f7fcb6444b440d1bd19ab547c47ae9b4f Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Wed, 8 Jan 2025 00:08:17 +0100 Subject: [PATCH 05/73] feat(vacuum): define AQUA_VACUUM_DAYS env var --- pkg/cli/util/util.go | 9 ++++++++- pkg/config/package.go | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/cli/util/util.go b/pkg/cli/util/util.go index 58c4cbaa1..418e25908 100644 --- a/pkg/cli/util/util.go +++ b/pkg/cli/util/util.go @@ -101,7 +101,7 @@ func setLogParams(c *cli.Context, param *config.Param, logE *logrus.Entry) { log.SetColor(param.LogColor, logE) } -func setEnvParams(param *config.Param) error { +func setEnvParams(param *config.Param) error { //nolint:cyclop if a := os.Getenv("AQUA_DISABLE_LAZY_INSTALL"); a != "" { disableLazyInstall, err := strconv.ParseBool(a) if err != nil { @@ -125,6 +125,13 @@ func setEnvParams(param *config.Param) error { } } } + if a := os.Getenv("AQUA_VACUUM_DAYS"); a != "" { + vacuumDays, err := strconv.Atoi(a) + if err != nil || vacuumDays <= 0 { + return fmt.Errorf("parse the environment variable AQUA_VACUUM_DAYS as a positive integer: %w", err) + } + param.VacuumDays = &vacuumDays + } return nil } diff --git a/pkg/config/package.go b/pkg/config/package.go index a4b2cb89b..96c87320f 100644 --- a/pkg/config/package.go +++ b/pkg/config/package.go @@ -294,6 +294,7 @@ type Param struct { Installed bool PolicyConfigFilePaths []string Commands []string + VacuumDays *int // When defined, vacuuming is enabled } func appendExt(s, format string) string { From a74ad33172e6ca27d2675cd825294d825704dd95 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Wed, 8 Jan 2025 00:11:26 +0100 Subject: [PATCH 06/73] feat(cli): add vacuum command for managing expired packages --- pkg/cli/runner.go | 2 + pkg/cli/vacuum/command.go | 116 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 pkg/cli/vacuum/command.go diff --git a/pkg/cli/runner.go b/pkg/cli/runner.go index ad31a6627..fc8946deb 100644 --- a/pkg/cli/runner.go +++ b/pkg/cli/runner.go @@ -20,6 +20,7 @@ import ( "github.com/aquaproj/aqua/v2/pkg/cli/update" "github.com/aquaproj/aqua/v2/pkg/cli/updateaqua" "github.com/aquaproj/aqua/v2/pkg/cli/util" + "github.com/aquaproj/aqua/v2/pkg/cli/vacuum" "github.com/aquaproj/aqua/v2/pkg/cli/version" "github.com/aquaproj/aqua/v2/pkg/cli/which" "github.com/urfave/cli/v2" @@ -105,6 +106,7 @@ func Run(ctx context.Context, param *util.Param, args ...string) error { //nolin upc.New, remove.New, update.New, + vacuum.New, ), } diff --git a/pkg/cli/vacuum/command.go b/pkg/cli/vacuum/command.go new file mode 100644 index 000000000..795ced7a2 --- /dev/null +++ b/pkg/cli/vacuum/command.go @@ -0,0 +1,116 @@ +package vacuum + +import ( + "errors" + "fmt" + "net/http" + + "github.com/aquaproj/aqua/v2/pkg/cli/profile" + "github.com/aquaproj/aqua/v2/pkg/cli/util" + "github.com/aquaproj/aqua/v2/pkg/config" + "github.com/aquaproj/aqua/v2/pkg/controller" + "github.com/urfave/cli/v2" +) + +const description = `Perform vacuuming tasks. +If no argument is provided, the vacuum will clean expired packages. + + # Execute vacuum cleaning + $ aqua vacuum + +This command has an alias "v". + + $ aqua v + +Enable vacuuming by setting the AQUA_VACUUM_DAYS environment variable to a value greater than 0. +This command removes versions of packages that have not been used for the specified number of days. + +You can list all packages managed by the vacuum system or only expired packages. + + # List all packages managed by the vacuum system + $ aqua vacuum --list + $ aqua vacuum -l + + # List only expired packages + $ aqua vacuum --expired + $ aqua vacuum -e + +` + +type command struct { + r *util.Param +} + +func New(r *util.Param) *cli.Command { + i := &command{ + r: r, + } + return &cli.Command{ + Name: "vacuum", + Usage: "Operate vacuuming tasks (If AQUA_VACUUM_DAYS is set)", + Aliases: []string{"v"}, + Description: description, + Action: i.action, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "list", + Usage: "List all packages managed by vacuum system", + Category: "list", + }, + &cli.BoolFlag{ + Name: "expired", + Usage: "List only expired packages", + Category: "list", + }, + }, + } +} + +// Define the vacuum modes for the CLI +const ( + ListPackages string = "list-packages" + ListExpiredPackages string = "list-expired-packages" + VacuumExpiredPackages string = "vacuum-expired-packages" +) + +func (i *command) action(c *cli.Context) error { + profiler, err := profile.Start(c) + if err != nil { + return fmt.Errorf("start CPU Profile or tracing: %w", err) + } + defer profiler.Stop() + + mode := parseVacuumMode(c) + + param := &config.Param{} + if err := util.SetParam(c, i.r.LogE, "vacuum", param, i.r.LDFlags); err != nil { + return fmt.Errorf("parse the command line arguments: %w", err) + } + + if param.VacuumDays == nil { + return errors.New("vacuum is not enabled, please set the AQUA_VACUUM_DAYS environment variable") + } + + ctrl := controller.InitializeVacuumCommandController(c.Context, param, http.DefaultClient, i.r.Runtime) + + vacuumMode, err := ctrl.GetVacuumModeCLI(mode) + if err != nil { + return fmt.Errorf("get vacuum mode: %w", err) + } + + if err := ctrl.Vacuum(c.Context, i.r.LogE, vacuumMode, nil); err != nil { + return fmt.Errorf("vacuum: %w", err) + } + return nil +} + +func parseVacuumMode(c *cli.Context) string { + mode := VacuumExpiredPackages + if c.Bool("list") { + mode = ListPackages + } + if c.Bool("expired") { + mode = ListExpiredPackages + } + return mode +} From 36950efbe904e5041fe2aad52b99259b44c2b4a3 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Wed, 8 Jan 2025 00:12:32 +0100 Subject: [PATCH 07/73] feat(installer): add vacuum controller integration --- pkg/installpackage/installer.go | 34 ++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/pkg/installpackage/installer.go b/pkg/installpackage/installer.go index 9aadb9251..18e435d1d 100644 --- a/pkg/installpackage/installer.go +++ b/pkg/installpackage/installer.go @@ -10,6 +10,7 @@ import ( "github.com/aquaproj/aqua/v2/pkg/config" "github.com/aquaproj/aqua/v2/pkg/config/aqua" "github.com/aquaproj/aqua/v2/pkg/config/registry" + "github.com/aquaproj/aqua/v2/pkg/controller/vacuum" "github.com/aquaproj/aqua/v2/pkg/cosign" "github.com/aquaproj/aqua/v2/pkg/download" "github.com/aquaproj/aqua/v2/pkg/ghattestation" @@ -57,9 +58,10 @@ type Installer struct { cosignDisabled bool slsaDisabled bool gaaDisabled bool + VacuumCtrl *vacuum.Controller } -func New(param *config.Param, downloader download.ClientAPI, rt *runtime.Runtime, fs afero.Fs, linker Linker, chkDL download.ChecksumDownloader, chkCalc ChecksumCalculator, unarchiver Unarchiver, cosignVerifier CosignVerifier, slsaVerifier SLSAVerifier, minisignVerifier MinisignVerifier, ghVerifier GitHubArtifactAttestationsVerifier, goInstallInstaller GoInstallInstaller, goBuildInstaller GoBuildInstaller, cargoPackageInstaller CargoPackageInstaller) *Installer { +func New(param *config.Param, downloader download.ClientAPI, rt *runtime.Runtime, fs afero.Fs, linker Linker, chkDL download.ChecksumDownloader, chkCalc ChecksumCalculator, unarchiver Unarchiver, cosignVerifier CosignVerifier, slsaVerifier SLSAVerifier, minisignVerifier MinisignVerifier, ghVerifier GitHubArtifactAttestationsVerifier, goInstallInstaller GoInstallInstaller, goBuildInstaller GoBuildInstaller, cargoPackageInstaller CargoPackageInstaller, opts ...Option) *Installer { ni := func(rt *runtime.Runtime) *Installer { return newInstaller(param, downloader, rt, fs, linker, chkDL, chkCalc, unarchiver, cosignVerifier, slsaVerifier, minisignVerifier, ghVerifier, goInstallInstaller, goBuildInstaller, cargoPackageInstaller) } @@ -84,6 +86,10 @@ func New(param *config.Param, downloader download.ClientAPI, rt *runtime.Runtime ghattestation.Package, ghattestation.Checksums(), ) + // Apply options + for _, opt := range opts { + opt(installer) + } return installer } @@ -115,6 +121,24 @@ func newInstaller(param *config.Param, downloader download.ClientAPI, rt *runtim } } +// ProvideControllerOptions creates the controller options +func ProvideControllerOptions(vacuumCtrl *vacuum.Controller) []Option { + if vacuumCtrl == nil { + return []Option{} + } + return []Option{WithVacuumController(vacuumCtrl)} +} + +// Option defines a function type for controller configuration +type Option func(*Installer) + +// WithVacuumController sets the VacuumController for the Installer +func WithVacuumController(vacuumCtrl *vacuum.Controller) Option { + return func(c *Installer) { + c.VacuumCtrl = vacuumCtrl + } +} + type Linker interface { Lstat(s string) (os.FileInfo, error) Symlink(dest, src string) error @@ -289,5 +313,13 @@ func (is *Installer) InstallPackage(ctx context.Context, logE *logrus.Entry, par return err } + // Optionally stores the package information in vacuum DB if vacuum controler is instantied and vacuum Enabled + if is.VacuumCtrl != nil && is.VacuumCtrl.IsVacuumEnabled(logE) { + logE.Debug("store package in vacuum") + if err := is.VacuumCtrl.Vacuum(ctx, logE, vacuum.AsyncStorePackage, []*config.Package{pkg}); err != nil { + logE.WithError(err).Error("store package in vacuum during install") + } + } + return is.checkFilesWrap(ctx, logE, param, pkgPath) } From 61308d6e48adf7dd3ca571641a801b773fd3fc49 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Wed, 8 Jan 2025 00:12:49 +0100 Subject: [PATCH 08/73] feat(controller): integrate vacuum controller into exec controller --- pkg/controller/exec/controller.go | 5 +- pkg/controller/exec/exec.go | 14 +++ pkg/controller/exec/exec_test.go | 169 ++++++++++++++++++++++++++---- 3 files changed, 168 insertions(+), 20 deletions(-) diff --git a/pkg/controller/exec/controller.go b/pkg/controller/exec/controller.go index be561e4e5..d4c1fec5c 100644 --- a/pkg/controller/exec/controller.go +++ b/pkg/controller/exec/controller.go @@ -7,6 +7,7 @@ import ( "runtime" "github.com/aquaproj/aqua/v2/pkg/config" + "github.com/aquaproj/aqua/v2/pkg/controller/vacuum" "github.com/aquaproj/aqua/v2/pkg/controller/which" "github.com/aquaproj/aqua/v2/pkg/installpackage" "github.com/aquaproj/aqua/v2/pkg/osexec" @@ -26,13 +27,14 @@ type Controller struct { fs afero.Fs policyReader PolicyReader enabledXSysExec bool + vacuumCtrl *vacuum.Controller } type Installer interface { InstallPackage(ctx context.Context, logE *logrus.Entry, param *installpackage.ParamInstallPackage) error } -func New(pkgInstaller Installer, whichCtrl WhichController, executor Executor, osEnv osenv.OSEnv, fs afero.Fs, policyReader PolicyReader) *Controller { +func New(pkgInstaller Installer, whichCtrl WhichController, executor Executor, osEnv osenv.OSEnv, fs afero.Fs, policyReader PolicyReader, vacuumCtrl *vacuum.Controller) *Controller { return &Controller{ stdin: os.Stdin, stdout: os.Stdout, @@ -43,6 +45,7 @@ func New(pkgInstaller Installer, whichCtrl WhichController, executor Executor, o enabledXSysExec: getEnabledXSysExec(osEnv, runtime.GOOS), fs: fs, policyReader: policyReader, + vacuumCtrl: vacuumCtrl, } } diff --git a/pkg/controller/exec/exec.go b/pkg/controller/exec/exec.go index 2e82f3773..f0a78ebb0 100644 --- a/pkg/controller/exec/exec.go +++ b/pkg/controller/exec/exec.go @@ -8,6 +8,7 @@ import ( "github.com/aquaproj/aqua/v2/pkg/checksum" "github.com/aquaproj/aqua/v2/pkg/config" + "github.com/aquaproj/aqua/v2/pkg/controller/vacuum" "github.com/aquaproj/aqua/v2/pkg/controller/which" "github.com/aquaproj/aqua/v2/pkg/installpackage" "github.com/aquaproj/aqua/v2/pkg/osexec" @@ -62,9 +63,22 @@ func (c *Controller) Exec(ctx context.Context, logE *logrus.Entry, param *config if err := c.install(ctx, logE, findResult, policyCfgs, param); err != nil { return err } + c.vacuumClose(ctx, logE) // Ensure that the vacuum process and db are closed return c.execCommandWithRetry(ctx, logE, findResult.ExePath, args...) } +func (c *Controller) vacuumClose(ctx context.Context, logE *logrus.Entry) { + if c.vacuumCtrl == nil { + return + } + if err := c.vacuumCtrl.Vacuum(ctx, logE, vacuum.Close, nil); err != nil { + // If the closing vacuum db failed, we should not stop the process + // so we log the error and continue the process. + // Updating vacuum db will be retried next time. + logE.WithError(err).Error("close the vacuum db failed") + } +} + func (c *Controller) install(ctx context.Context, logE *logrus.Entry, findResult *which.FindResult, policies []*policy.Config, param *config.Param) error { var checksums *checksum.Checksums if findResult.Config.ChecksumEnabled(param.EnforceChecksum, param.Checksum) { diff --git a/pkg/controller/exec/exec_test.go b/pkg/controller/exec/exec_test.go index ac08b8c47..f7376c40a 100644 --- a/pkg/controller/exec/exec_test.go +++ b/pkg/controller/exec/exec_test.go @@ -14,6 +14,7 @@ import ( finder "github.com/aquaproj/aqua/v2/pkg/config-finder" reader "github.com/aquaproj/aqua/v2/pkg/config-reader" execCtrl "github.com/aquaproj/aqua/v2/pkg/controller/exec" + "github.com/aquaproj/aqua/v2/pkg/controller/vacuum" "github.com/aquaproj/aqua/v2/pkg/controller/which" "github.com/aquaproj/aqua/v2/pkg/cosign" "github.com/aquaproj/aqua/v2/pkg/download" @@ -28,7 +29,6 @@ import ( "github.com/aquaproj/aqua/v2/pkg/testutil" "github.com/aquaproj/aqua/v2/pkg/unarchive" "github.com/sirupsen/logrus" - "github.com/spf13/afero" "github.com/suzuki-shunsuke/go-osenv/osenv" ) @@ -131,6 +131,56 @@ packages: "../foo/gh": "/usr/local/bin/gh", }, }, + { + name: "vacuumEnabled", + rt: &runtime.Runtime{ + GOOS: "linux", + GOARCH: "amd64", + }, + param: &config.Param{ + PWD: "/home/foo/workspace", + ConfigFilePath: "aqua.yaml", + RootDir: "/home/foo/.local/share/aquaproj-aqua", + MaxParallelism: 5, + VacuumDays: func(i int) *int { return &i }(1), + }, + exeName: "aqua-installer", + dirs: []string{ + "/home/foo/workspace/.git", + }, + files: map[string]string{ + "/home/foo/workspace/aqua.yaml": `registries: +- type: local + name: standard + path: registry.yaml +packages: +- name: aquaproj/aqua-installer@v1.0.0 +`, + "/home/foo/workspace/registry.yaml": `packages: +- type: github_content + repo_owner: aquaproj + repo_name: aqua-installer + path: aqua-installer +`, + "/home/foo/.local/share/aquaproj-aqua/pkgs/github_content/github.com/aquaproj/aqua-installer/v1.0.0/aqua-installer/aqua-installer": "", + "/home/foo/workspace/aqua-policy.yaml": ` +registries: +- type: local + name: standard + path: registry.yaml +packages: +- type: local +`, + "/home/foo/.local/share/aquaproj-aqua/policies/home/foo/workspace/aqua-policy.yaml": ` +registries: +- type: local + name: standard + path: registry.yaml +packages: +- type: local +`, + }, + }, } logE := logrus.NewEntry(logrus.New()) ctx := context.Background() @@ -154,7 +204,8 @@ packages: executor := &osexec.Mock{} pkgInstaller := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}) policyFinder := policy.NewConfigFinder(fs) - ctrl := execCtrl.New(pkgInstaller, whichCtrl, executor, osEnv, fs, policy.NewReader(fs, policy.NewValidator(d.param, fs), policyFinder, policy.NewConfigReader(fs))) + vacuumCtrl := vacuum.New(d.param, fs) + ctrl := execCtrl.New(pkgInstaller, whichCtrl, executor, osEnv, fs, policy.NewReader(fs, policy.NewValidator(d.param, fs), policyFinder, policy.NewConfigReader(fs)), vacuumCtrl) if err := ctrl.Exec(ctx, logE, d.param, d.exeName, d.args...); err != nil { if d.isErr { return @@ -189,10 +240,11 @@ func downloadTestFile(uri, tempDir string) (string, error) { return filePath, nil } -func Benchmark_controller_Exec(b *testing.B) { //nolint:funlen,gocognit +func Benchmark_controller_Exec(b *testing.B) { //nolint:funlen data := []struct { name string files map[string]string + dirs []string links map[string]string env map[string]string param *config.Param @@ -209,30 +261,107 @@ func Benchmark_controller_Exec(b *testing.B) { //nolint:funlen,gocognit }, param: &config.Param{ PWD: "/home/foo/workspace", + ConfigFilePath: "aqua.yaml", RootDir: "/home/foo/.local/share/aquaproj-aqua", MaxParallelism: 5, }, exeName: "aqua-installer", - files: map[string]string{}, + dirs: []string{ + "/home/foo/workspace/.git", + }, + files: map[string]string{ + "/home/foo/workspace/aqua.yaml": `registries: +- type: local + name: standard + path: registry.yaml +packages: +- name: aquaproj/aqua-installer@v1.0.0 +`, + "/home/foo/workspace/registry.yaml": `packages: +- type: github_content + repo_owner: aquaproj + repo_name: aqua-installer + path: aqua-installer +`, + "/home/foo/.local/share/aquaproj-aqua/pkgs/github_content/github.com/aquaproj/aqua-installer/v1.0.0/aqua-installer/aqua-installer": "", + "/home/foo/workspace/aqua-policy.yaml": ` +registries: +- type: local + name: standard + path: registry.yaml +packages: +- type: local +`, + "/home/foo/.local/share/aquaproj-aqua/policies/home/foo/workspace/aqua-policy.yaml": ` +registries: +- type: local + name: standard + path: registry.yaml +packages: +- type: local +`, + }, }, - } - logE := logrus.NewEntry(logrus.New()) - ctx := context.Background() - for _, d := range data { - b.Run("normal", func(b *testing.B) { - tempDir := b.TempDir() - d.param.ConfigFilePath = filepath.Join(tempDir, "aqua.yaml") - d.files[d.param.ConfigFilePath] = `registries: + { + name: "vacuumEnabled", + rt: &runtime.Runtime{ + GOOS: "linux", + GOARCH: "amd64", + }, + param: &config.Param{ + PWD: "/home/foo/workspace", + ConfigFilePath: "aqua.yaml", + RootDir: "/home/foo/.local/share/aquaproj-aqua", + MaxParallelism: 5, + VacuumDays: func(i int) *int { return &i }(1), + }, + exeName: "aqua-installer", + env: map[string]string{ + "AQUA_VACUUM_DAYS": "1", + }, + dirs: []string{ + "/home/foo/workspace/.git", + }, + files: map[string]string{ + "/home/foo/workspace/aqua.yaml": `registries: - type: local name: standard path: registry.yaml packages: - name: aquaproj/aqua-installer@v1.0.0 -` - if _, err := downloadTestFile("https://raw.githubusercontent.com/aquaproj/aqua-registry/v2.19.0/registry.yaml", tempDir); err != nil { - b.Fatal(err) - } - fs, err := testutil.NewFs(d.files) +`, + "/home/foo/workspace/registry.yaml": `packages: +- type: github_content + repo_owner: aquaproj + repo_name: aqua-installer + path: aqua-installer +`, + "/home/foo/.local/share/aquaproj-aqua/pkgs/github_content/github.com/aquaproj/aqua-installer/v1.0.0/aqua-installer/aqua-installer": "", + "/home/foo/workspace/aqua-policy.yaml": ` +registries: +- type: local + name: standard + path: registry.yaml +packages: +- type: local +`, + "/home/foo/.local/share/aquaproj-aqua/policies/home/foo/workspace/aqua-policy.yaml": ` +registries: +- type: local + name: standard + path: registry.yaml +packages: +- type: local +`, + }, + }, + } + + logE := logrus.NewEntry(logrus.New()) + ctx := context.Background() + for _, d := range data { + b.Run(d.name, func(b *testing.B) { + fs, err := testutil.NewFs(d.files, d.dirs...) if err != nil { b.Fatal(err) } @@ -244,11 +373,13 @@ packages: } ghDownloader := download.NewGitHubContentFileDownloader(nil, download.NewHTTPDownloader(http.DefaultClient)) osEnv := osenv.NewMock(d.env) - whichCtrl := which.New(d.param, finder.NewConfigFinder(fs), reader.New(fs, d.param), registry.New(d.param, ghDownloader, afero.NewOsFs(), d.rt, &cosign.MockVerifier{}, &slsa.MockVerifier{}), d.rt, osEnv, fs, linker) + whichCtrl := which.New(d.param, finder.NewConfigFinder(fs), reader.New(fs, d.param), registry.New(d.param, ghDownloader, fs, d.rt, &cosign.MockVerifier{}, &slsa.MockVerifier{}), d.rt, osEnv, fs, linker) downloader := download.NewDownloader(nil, download.NewHTTPDownloader(http.DefaultClient)) executor := &osexec.Mock{} pkgInstaller := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}) - ctrl := execCtrl.New(pkgInstaller, whichCtrl, executor, osEnv, fs, &policy.MockReader{}) + policyFinder := policy.NewConfigFinder(fs) + vacuumCtrl := vacuum.New(d.param, fs) + ctrl := execCtrl.New(pkgInstaller, whichCtrl, executor, osEnv, fs, policy.NewReader(fs, policy.NewValidator(d.param, fs), policyFinder, policy.NewConfigReader(fs)), vacuumCtrl) b.ResetTimer() for range b.N { func() { From c5858fde2b0a28e69a668fd84b84f040e403cacb Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Wed, 8 Jan 2025 00:13:10 +0100 Subject: [PATCH 09/73] feat(controller): add vacuum command controller initialization --- pkg/controller/wire.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pkg/controller/wire.go b/pkg/controller/wire.go index ffc463a6d..be501b403 100644 --- a/pkg/controller/wire.go +++ b/pkg/controller/wire.go @@ -29,6 +29,7 @@ import ( "github.com/aquaproj/aqua/v2/pkg/controller/update" "github.com/aquaproj/aqua/v2/pkg/controller/updateaqua" "github.com/aquaproj/aqua/v2/pkg/controller/updatechecksum" + "github.com/aquaproj/aqua/v2/pkg/controller/vacuum" "github.com/aquaproj/aqua/v2/pkg/controller/which" "github.com/aquaproj/aqua/v2/pkg/cosign" "github.com/aquaproj/aqua/v2/pkg/domain" @@ -254,6 +255,7 @@ func InitializeInstallCommandController(ctx context.Context, param *config.Param ), wire.NewSet( installpackage.New, + installpackage.ProvideControllerOptions, wire.Bind(new(install.Installer), new(*installpackage.Installer)), ), wire.NewSet( @@ -348,6 +350,7 @@ func InitializeInstallCommandController(ctx context.Context, param *config.Param installpackage.NewCargoPackageInstallerImpl, wire.Bind(new(installpackage.CargoPackageInstaller), new(*installpackage.CargoPackageInstallerImpl)), ), + vacuum.New, ) return &install.Controller{}, nil } @@ -424,6 +427,7 @@ func InitializeExecCommandController(ctx context.Context, param *config.Param, h ), wire.NewSet( installpackage.New, + installpackage.ProvideControllerOptions, wire.Bind(new(cexec.Installer), new(*installpackage.Installer)), ), wire.NewSet( @@ -538,6 +542,7 @@ func InitializeExecCommandController(ctx context.Context, param *config.Param, h installpackage.NewCargoPackageInstallerImpl, wire.Bind(new(installpackage.CargoPackageInstaller), new(*installpackage.CargoPackageInstallerImpl)), ), + vacuum.New, ) return &cexec.Controller{}, nil } @@ -556,6 +561,7 @@ func InitializeUpdateAquaCommandController(ctx context.Context, param *config.Pa ), wire.NewSet( installpackage.New, + installpackage.ProvideControllerOptions, wire.Bind(new(updateaqua.AquaInstaller), new(*installpackage.Installer)), ), download.NewHTTPDownloader, @@ -630,6 +636,7 @@ func InitializeUpdateAquaCommandController(ctx context.Context, param *config.Pa installpackage.NewCargoPackageInstallerImpl, wire.Bind(new(installpackage.CargoPackageInstaller), new(*installpackage.CargoPackageInstallerImpl)), ), + provideNilVacuumController, // vacuum controller is not used so we provide nil but it is required by installPackage ) return &updateaqua.Controller{}, nil } @@ -652,6 +659,7 @@ func InitializeCopyCommandController(ctx context.Context, param *config.Param, h ), wire.NewSet( installpackage.New, + installpackage.ProvideControllerOptions, wire.Bind(new(install.Installer), new(*installpackage.Installer)), wire.Bind(new(cp.PackageInstaller), new(*installpackage.Installer)), ), @@ -770,6 +778,7 @@ func InitializeCopyCommandController(ctx context.Context, param *config.Param, h installpackage.NewCargoPackageInstallerImpl, wire.Bind(new(installpackage.CargoPackageInstaller), new(*installpackage.CargoPackageInstallerImpl)), ), + provideNilVacuumController, // vacuum controller is not used so we provide nil but it is required by installer and installpackage ) return &cp.Controller{}, nil } @@ -1040,3 +1049,15 @@ func InitializeRemoveCommandController(ctx context.Context, param *config.Param, ) return &remove.Controller{} } + +func InitializeVacuumCommandController(ctx context.Context, param *config.Param, httpClient *http.Client, rt *runtime.Runtime) *vacuum.Controller { + wire.Build( + vacuum.New, + afero.NewOsFs, + ) + return &vacuum.Controller{} +} + +func provideNilVacuumController() *vacuum.Controller { + return nil +} From 6a0d6616fd8d7519a4523f6682ee3c9986b54a13 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Wed, 8 Jan 2025 00:13:40 +0100 Subject: [PATCH 10/73] chore(wire): cmdx wire to regenerate it --- pkg/controller/wire_gen.go | 45 ++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/pkg/controller/wire_gen.go b/pkg/controller/wire_gen.go index 931b9a7fb..bcdb08198 100644 --- a/pkg/controller/wire_gen.go +++ b/pkg/controller/wire_gen.go @@ -29,6 +29,7 @@ import ( "github.com/aquaproj/aqua/v2/pkg/controller/update" "github.com/aquaproj/aqua/v2/pkg/controller/updateaqua" "github.com/aquaproj/aqua/v2/pkg/controller/updatechecksum" + "github.com/aquaproj/aqua/v2/pkg/controller/vacuum" "github.com/aquaproj/aqua/v2/pkg/controller/which" "github.com/aquaproj/aqua/v2/pkg/cosign" "github.com/aquaproj/aqua/v2/pkg/download" @@ -149,13 +150,15 @@ func InitializeInstallCommandController(ctx context.Context, param *config.Param goInstallInstallerImpl := installpackage.NewGoInstallInstallerImpl(executor) goBuildInstallerImpl := installpackage.NewGoBuildInstallerImpl(executor) cargoPackageInstallerImpl := installpackage.NewCargoPackageInstallerImpl(executor, fs) - installpackageInstaller := installpackage.New(param, downloader, rt, fs, linker, checksumDownloaderImpl, calculator, unarchiver, verifier, slsaVerifier, minisignVerifier, ghattestationVerifier, goInstallInstallerImpl, goBuildInstallerImpl, cargoPackageInstallerImpl) + controller := vacuum.New(param, fs) + v := installpackage.ProvideControllerOptions(controller) + installpackageInstaller := installpackage.New(param, downloader, rt, fs, linker, checksumDownloaderImpl, calculator, unarchiver, verifier, slsaVerifier, minisignVerifier, ghattestationVerifier, goInstallInstallerImpl, goBuildInstallerImpl, cargoPackageInstallerImpl, v...) validatorImpl := policy.NewValidator(param, fs) configFinderImpl := policy.NewConfigFinder(fs) configReaderImpl := policy.NewConfigReader(fs) policyReader := policy.NewReader(fs, validatorImpl, configFinderImpl, configReaderImpl) - controller := install.New(param, configFinder, configReader, installer, installpackageInstaller, fs, rt, policyReader) - return controller, nil + installController := install.New(param, configFinder, configReader, installer, installpackageInstaller, fs, rt, policyReader) + return installController, nil } func InitializeWhichCommandController(ctx context.Context, param *config.Param, httpClient *http.Client, rt *runtime.Runtime) *which.Controller { @@ -203,18 +206,20 @@ func InitializeExecCommandController(ctx context.Context, param *config.Param, h goInstallInstallerImpl := installpackage.NewGoInstallInstallerImpl(executor) goBuildInstallerImpl := installpackage.NewGoBuildInstallerImpl(executor) cargoPackageInstallerImpl := installpackage.NewCargoPackageInstallerImpl(executor, fs) - installer := installpackage.New(param, downloader, rt, fs, linker, checksumDownloaderImpl, calculator, unarchiver, verifier, slsaVerifier, minisignVerifier, ghattestationVerifier, goInstallInstallerImpl, goBuildInstallerImpl, cargoPackageInstallerImpl) + controller := vacuum.New(param, fs) + v := installpackage.ProvideControllerOptions(controller) + installer := installpackage.New(param, downloader, rt, fs, linker, checksumDownloaderImpl, calculator, unarchiver, verifier, slsaVerifier, minisignVerifier, ghattestationVerifier, goInstallInstallerImpl, goBuildInstallerImpl, cargoPackageInstallerImpl, v...) configFinder := finder.NewConfigFinder(fs) configReader := reader.New(fs, param) gitHubContentFileDownloader := download.NewGitHubContentFileDownloader(repositoriesService, httpDownloader) registryInstaller := registry.New(param, gitHubContentFileDownloader, fs, rt, verifier, slsaVerifier) osEnv := osenv.New() - controller := which.New(param, configFinder, configReader, registryInstaller, rt, osEnv, fs, linker) + whichController := which.New(param, configFinder, configReader, registryInstaller, rt, osEnv, fs, linker) validatorImpl := policy.NewValidator(param, fs) configFinderImpl := policy.NewConfigFinder(fs) configReaderImpl := policy.NewConfigReader(fs) policyReader := policy.NewReader(fs, validatorImpl, configFinderImpl, configReaderImpl) - execController := exec.New(installer, controller, executor, osEnv, fs, policyReader) + execController := exec.New(installer, whichController, executor, osEnv, fs, policyReader, controller) return execController, nil } @@ -244,9 +249,11 @@ func InitializeUpdateAquaCommandController(ctx context.Context, param *config.Pa goInstallInstallerImpl := installpackage.NewGoInstallInstallerImpl(executor) goBuildInstallerImpl := installpackage.NewGoBuildInstallerImpl(executor) cargoPackageInstallerImpl := installpackage.NewCargoPackageInstallerImpl(executor, fs) - installer := installpackage.New(param, downloader, rt, fs, linker, checksumDownloaderImpl, calculator, unarchiver, verifier, slsaVerifier, minisignVerifier, ghattestationVerifier, goInstallInstallerImpl, goBuildInstallerImpl, cargoPackageInstallerImpl) - controller := updateaqua.New(param, fs, rt, repositoriesService, installer) - return controller, nil + controller := provideNilVacuumController() + v := installpackage.ProvideControllerOptions(controller) + installer := installpackage.New(param, downloader, rt, fs, linker, checksumDownloaderImpl, calculator, unarchiver, verifier, slsaVerifier, minisignVerifier, ghattestationVerifier, goInstallInstallerImpl, goBuildInstallerImpl, cargoPackageInstallerImpl, v...) + updateaquaController := updateaqua.New(param, fs, rt, repositoriesService, installer) + return updateaquaController, nil } func InitializeCopyCommandController(ctx context.Context, param *config.Param, httpClient *http.Client, rt *runtime.Runtime) (*cp.Controller, error) { @@ -275,19 +282,21 @@ func InitializeCopyCommandController(ctx context.Context, param *config.Param, h goInstallInstallerImpl := installpackage.NewGoInstallInstallerImpl(executor) goBuildInstallerImpl := installpackage.NewGoBuildInstallerImpl(executor) cargoPackageInstallerImpl := installpackage.NewCargoPackageInstallerImpl(executor, fs) - installer := installpackage.New(param, downloader, rt, fs, linker, checksumDownloaderImpl, calculator, unarchiver, verifier, slsaVerifier, minisignVerifier, ghattestationVerifier, goInstallInstallerImpl, goBuildInstallerImpl, cargoPackageInstallerImpl) + controller := provideNilVacuumController() + v := installpackage.ProvideControllerOptions(controller) + installer := installpackage.New(param, downloader, rt, fs, linker, checksumDownloaderImpl, calculator, unarchiver, verifier, slsaVerifier, minisignVerifier, ghattestationVerifier, goInstallInstallerImpl, goBuildInstallerImpl, cargoPackageInstallerImpl, v...) configFinder := finder.NewConfigFinder(fs) configReader := reader.New(fs, param) gitHubContentFileDownloader := download.NewGitHubContentFileDownloader(repositoriesService, httpDownloader) registryInstaller := registry.New(param, gitHubContentFileDownloader, fs, rt, verifier, slsaVerifier) osEnv := osenv.New() - controller := which.New(param, configFinder, configReader, registryInstaller, rt, osEnv, fs, linker) + whichController := which.New(param, configFinder, configReader, registryInstaller, rt, osEnv, fs, linker) validatorImpl := policy.NewValidator(param, fs) configFinderImpl := policy.NewConfigFinder(fs) configReaderImpl := policy.NewConfigReader(fs) policyReader := policy.NewReader(fs, validatorImpl, configFinderImpl, configReaderImpl) installController := install.New(param, configFinder, configReader, registryInstaller, installer, fs, rt, policyReader) - cpController := cp.New(param, installer, fs, rt, controller, installController, policyReader) + cpController := cp.New(param, installer, fs, rt, whichController, installController, policyReader) return cpController, nil } @@ -381,3 +390,15 @@ func InitializeRemoveCommandController(ctx context.Context, param *config.Param, removeController := remove.New(param, target, fs, rt, configFinder, configReader, installer, fuzzyfinderFinder, controller) return removeController } + +func InitializeVacuumCommandController(ctx context.Context, param *config.Param, httpClient *http.Client, rt *runtime.Runtime) *vacuum.Controller { + fs := afero.NewOsFs() + controller := vacuum.New(param, fs) + return controller +} + +// wire.go: + +func provideNilVacuumController() *vacuum.Controller { + return nil +} From cf972231818bae947018b59b8d87d8f7374d42ba Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Wed, 8 Jan 2025 07:42:32 +0100 Subject: [PATCH 11/73] test(exec): remove unused function --- pkg/controller/exec/exec_test.go | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/pkg/controller/exec/exec_test.go b/pkg/controller/exec/exec_test.go index f7376c40a..636e06025 100644 --- a/pkg/controller/exec/exec_test.go +++ b/pkg/controller/exec/exec_test.go @@ -2,11 +2,7 @@ package exec_test import ( "context" - "fmt" - "io" "net/http" - "os" - "path/filepath" "testing" "github.com/aquaproj/aqua/v2/pkg/checksum" @@ -219,27 +215,6 @@ packages: } } -func downloadTestFile(uri, tempDir string) (string, error) { - req, err := http.NewRequest(http.MethodGet, uri, nil) //nolint:noctx - if err != nil { - return "", fmt.Errorf("create a request: %w", err) - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return "", fmt.Errorf("send a HTTP request: %w", err) - } - defer resp.Body.Close() - filePath := filepath.Join(tempDir, "registry.yaml") - f, err := os.Create(filePath) - if err != nil { - return "", fmt.Errorf("create a file: %w", err) - } - if _, err := io.Copy(f, resp.Body); err != nil { - return "", fmt.Errorf("write a response body to a file: %w", err) - } - return filePath, nil -} - func Benchmark_controller_Exec(b *testing.B) { //nolint:funlen data := []struct { name string From 62066f434b9a6cd2b95c25b2bfb84e5ce7e37e2f Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Wed, 8 Jan 2025 10:09:21 +0100 Subject: [PATCH 12/73] chore(vacuum): make newStoreQueue private function --- pkg/controller/vacuum/controller.go | 2 +- pkg/controller/vacuum/queue_store.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/controller/vacuum/controller.go b/pkg/controller/vacuum/controller.go index b1c4cbee4..67a82478c 100644 --- a/pkg/controller/vacuum/controller.go +++ b/pkg/controller/vacuum/controller.go @@ -30,7 +30,7 @@ func New(param *config.Param, fs afero.Fs) *Controller { Param: param, fs: fs, } - vc.storeQueue = NewStoreQueue(vc) + vc.storeQueue = newStoreQueue(vc) return vc } diff --git a/pkg/controller/vacuum/queue_store.go b/pkg/controller/vacuum/queue_store.go index 8d1aae246..b69631472 100644 --- a/pkg/controller/vacuum/queue_store.go +++ b/pkg/controller/vacuum/queue_store.go @@ -21,8 +21,8 @@ type StoreQueue struct { closeOnce sync.Once } -// NewStoreQueue initializes the task queue with a single worker. -func NewStoreQueue(vc *Controller) *StoreQueue { +// newStoreQueue initializes the task queue with a single worker. +func newStoreQueue(vc *Controller) *StoreQueue { const maxTasks = 100 sq := &StoreQueue{ taskQueue: make(chan StoreRequest, maxTasks), From 046ab6e84e67baaff3bc65863bc9f1429a19d78d Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Wed, 8 Jan 2025 10:20:23 +0100 Subject: [PATCH 13/73] chore(install): defer close vacuum to ensure async finalized --- pkg/controller/install/controller.go | 5 ++++- pkg/controller/install/install.go | 14 ++++++++++++++ pkg/controller/install/install_test.go | 4 +++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/pkg/controller/install/controller.go b/pkg/controller/install/controller.go index 66cf88f00..05d2c1523 100644 --- a/pkg/controller/install/controller.go +++ b/pkg/controller/install/controller.go @@ -7,6 +7,7 @@ import ( "github.com/aquaproj/aqua/v2/pkg/config" "github.com/aquaproj/aqua/v2/pkg/config/aqua" "github.com/aquaproj/aqua/v2/pkg/config/registry" + "github.com/aquaproj/aqua/v2/pkg/controller/vacuum" "github.com/aquaproj/aqua/v2/pkg/installpackage" "github.com/aquaproj/aqua/v2/pkg/policy" "github.com/aquaproj/aqua/v2/pkg/runtime" @@ -26,9 +27,10 @@ type Controller struct { excludedTags map[string]struct{} policyReader PolicyReader skipLink bool + vacuumCtrl *vacuum.Controller } -func New(param *config.Param, configFinder ConfigFinder, configReader ConfigReader, registInstaller RegistryInstaller, pkgInstaller Installer, fs afero.Fs, rt *runtime.Runtime, policyReader PolicyReader) *Controller { +func New(param *config.Param, configFinder ConfigFinder, configReader ConfigReader, registInstaller RegistryInstaller, pkgInstaller Installer, fs afero.Fs, rt *runtime.Runtime, policyReader PolicyReader, vacuumCtrl *vacuum.Controller) *Controller { return &Controller{ rootDir: param.RootDir, configFinder: configFinder, @@ -41,6 +43,7 @@ func New(param *config.Param, configFinder ConfigFinder, configReader ConfigRead tags: param.Tags, excludedTags: param.ExcludedTags, policyReader: policyReader, + vacuumCtrl: vacuumCtrl, } } diff --git a/pkg/controller/install/install.go b/pkg/controller/install/install.go index ef1d8a88e..026b8f71e 100644 --- a/pkg/controller/install/install.go +++ b/pkg/controller/install/install.go @@ -9,6 +9,7 @@ import ( "github.com/aquaproj/aqua/v2/pkg/config" finder "github.com/aquaproj/aqua/v2/pkg/config-finder" "github.com/aquaproj/aqua/v2/pkg/config/aqua" + "github.com/aquaproj/aqua/v2/pkg/controller/vacuum" "github.com/aquaproj/aqua/v2/pkg/installpackage" "github.com/aquaproj/aqua/v2/pkg/osfile" "github.com/aquaproj/aqua/v2/pkg/policy" @@ -85,6 +86,7 @@ func (c *Controller) installAll(ctx context.Context, logE *logrus.Entry, param * } func (c *Controller) install(ctx context.Context, logE *logrus.Entry, cfgFilePath string, policyConfigs []*policy.Config, param *config.Param) error { + defer c.vacuumClose(ctx, logE) cfg := &aqua.Config{} if cfgFilePath == "" { return finder.ErrConfigFileNotFound @@ -128,3 +130,15 @@ func (c *Controller) install(ctx context.Context, logE *logrus.Entry, cfgFilePat DisablePolicy: param.DisablePolicy, }) } + +func (c *Controller) vacuumClose(ctx context.Context, logE *logrus.Entry) { + if c.vacuumCtrl == nil { + return + } + if err := c.vacuumCtrl.Vacuum(ctx, logE, vacuum.Close, nil); err != nil { + // If the closing vacuum db failed, we should not stop the process + // so we log the error and continue the process. + // Updating vacuum db will be retried next time. + logE.WithError(err).Error("close the vacuum db failed") + } +} diff --git a/pkg/controller/install/install_test.go b/pkg/controller/install/install_test.go index 7c9692737..f0ffe040b 100644 --- a/pkg/controller/install/install_test.go +++ b/pkg/controller/install/install_test.go @@ -11,6 +11,7 @@ import ( finder "github.com/aquaproj/aqua/v2/pkg/config-finder" reader "github.com/aquaproj/aqua/v2/pkg/config-reader" "github.com/aquaproj/aqua/v2/pkg/controller/install" + "github.com/aquaproj/aqua/v2/pkg/controller/vacuum" "github.com/aquaproj/aqua/v2/pkg/cosign" "github.com/aquaproj/aqua/v2/pkg/download" "github.com/aquaproj/aqua/v2/pkg/ghattestation" @@ -106,7 +107,8 @@ packages: pkgInstaller := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}) policyFinder := policy.NewConfigFinder(fs) policyReader := policy.NewReader(fs, &policy.MockValidator{}, policyFinder, policy.NewConfigReader(fs)) - ctrl := install.New(d.param, finder.NewConfigFinder(fs), reader.New(fs, d.param), registry.New(d.param, registryDownloader, fs, d.rt, &cosign.MockVerifier{}, &slsa.MockVerifier{}), pkgInstaller, fs, d.rt, policyReader) + vacuum := vacuum.New(d.param, fs) + ctrl := install.New(d.param, finder.NewConfigFinder(fs), reader.New(fs, d.param), registry.New(d.param, registryDownloader, fs, d.rt, &cosign.MockVerifier{}, &slsa.MockVerifier{}), pkgInstaller, fs, d.rt, policyReader, vacuum) if err := ctrl.Install(ctx, logE, d.param); err != nil { if d.isErr { return From 01624442094414a5344e727edf9bc377f7100564 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Fri, 10 Jan 2025 14:03:50 +0100 Subject: [PATCH 14/73] chore(vacuum): use int instead of pointer --- pkg/cli/util/util.go | 10 +++++++--- pkg/config/package.go | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/cli/util/util.go b/pkg/cli/util/util.go index 418e25908..265294da9 100644 --- a/pkg/cli/util/util.go +++ b/pkg/cli/util/util.go @@ -125,12 +125,16 @@ func setEnvParams(param *config.Param) error { //nolint:cyclop } } } + param.VacuumDays = 0 // Disabled by default if a := os.Getenv("AQUA_VACUUM_DAYS"); a != "" { vacuumDays, err := strconv.Atoi(a) - if err != nil || vacuumDays <= 0 { - return fmt.Errorf("parse the environment variable AQUA_VACUUM_DAYS as a positive integer: %w", err) + if err != nil { + return fmt.Errorf("parse the environment variable AQUA_VACUUM_DAYS as an integer: %w", err) + } + if vacuumDays <= 0 { + return fmt.Errorf("the environment variable AQUA_VACUUM_DAYS must be a positive integer: %d", vacuumDays) } - param.VacuumDays = &vacuumDays + param.VacuumDays = vacuumDays } return nil } diff --git a/pkg/config/package.go b/pkg/config/package.go index 96c87320f..630ce6a35 100644 --- a/pkg/config/package.go +++ b/pkg/config/package.go @@ -294,7 +294,7 @@ type Param struct { Installed bool PolicyConfigFilePaths []string Commands []string - VacuumDays *int // When defined, vacuuming is enabled + VacuumDays int } func appendExt(s, format string) string { From 77bcc0e196d4d5adb680751ab53543a19d1b98ff Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Fri, 10 Jan 2025 14:04:08 +0100 Subject: [PATCH 15/73] chore(vacuum): refactor CLI commands for vacuum operations --- pkg/cli/vacuum/command.go | 74 ++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 40 deletions(-) diff --git a/pkg/cli/vacuum/command.go b/pkg/cli/vacuum/command.go index 795ced7a2..ac1f7d9cc 100644 --- a/pkg/cli/vacuum/command.go +++ b/pkg/cli/vacuum/command.go @@ -28,13 +28,14 @@ This command removes versions of packages that have not been used for the specif You can list all packages managed by the vacuum system or only expired packages. # List all packages managed by the vacuum system - $ aqua vacuum --list - $ aqua vacuum -l + $ aqua vacuum show # List only expired packages - $ aqua vacuum --expired - $ aqua vacuum -e + $ aqua vacuum show --expired + $ aqua vacuum show -e + # Run vacuum cleaning + $ aqua vacuum run ` type command struct { @@ -50,29 +51,30 @@ func New(r *util.Param) *cli.Command { Usage: "Operate vacuuming tasks (If AQUA_VACUUM_DAYS is set)", Aliases: []string{"v"}, Description: description, - Action: i.action, - Flags: []cli.Flag{ - &cli.BoolFlag{ - Name: "list", - Usage: "List all packages managed by vacuum system", - Category: "list", + Subcommands: []*cli.Command{ + { + Name: "show", + Aliases: []string{"s"}, + Usage: "Show packages managed by vacuum system", + Action: i.action, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "expired", + Usage: "Show only expired packages", + Aliases: []string{"e"}, + }, + }, }, - &cli.BoolFlag{ - Name: "expired", - Usage: "List only expired packages", - Category: "list", + { + Name: "run", + Aliases: []string{"r"}, + Usage: "Run vacuum cleaning", + Action: i.action, }, }, } } -// Define the vacuum modes for the CLI -const ( - ListPackages string = "list-packages" - ListExpiredPackages string = "list-expired-packages" - VacuumExpiredPackages string = "vacuum-expired-packages" -) - func (i *command) action(c *cli.Context) error { profiler, err := profile.Start(c) if err != nil { @@ -80,37 +82,29 @@ func (i *command) action(c *cli.Context) error { } defer profiler.Stop() - mode := parseVacuumMode(c) - param := &config.Param{} if err := util.SetParam(c, i.r.LogE, "vacuum", param, i.r.LDFlags); err != nil { return fmt.Errorf("parse the command line arguments: %w", err) } - if param.VacuumDays == nil { + if param.VacuumDays == 0 { return errors.New("vacuum is not enabled, please set the AQUA_VACUUM_DAYS environment variable") } ctrl := controller.InitializeVacuumCommandController(c.Context, param, http.DefaultClient, i.r.Runtime) - vacuumMode, err := ctrl.GetVacuumModeCLI(mode) - if err != nil { - return fmt.Errorf("get vacuum mode: %w", err) + if c.Command.Name == "show" { + if err := ctrl.ListPackages(i.r.LogE, c.Bool("expired")); err != nil { + return fmt.Errorf("list packages: %w", err) + } + return nil } - if err := ctrl.Vacuum(c.Context, i.r.LogE, vacuumMode, nil); err != nil { - return fmt.Errorf("vacuum: %w", err) + if c.Command.Name == "run" { + if err := ctrl.Vacuum(i.r.LogE); err != nil { + return fmt.Errorf("run: %w", err) + } } - return nil -} -func parseVacuumMode(c *cli.Context) string { - mode := VacuumExpiredPackages - if c.Bool("list") { - mode = ListPackages - } - if c.Bool("expired") { - mode = ListExpiredPackages - } - return mode + return nil } From f09bf45fd7b2f6a3547c95ba24f8c03e684a7909 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Fri, 10 Jan 2025 14:08:57 +0100 Subject: [PATCH 16/73] chore(vacuum): use public functions instead of mode Refactored code to exclusively use `StorePackage` as asynchronous functions. It is important to note that from now on, **you must ensure all tasks are finalized by calling the `Close` functions after invoking `StorePackage`.** Replaced the generated key with `pkgPath` to ensure uniqueness. Changed `pkgPath` type from an array of strings to a single string. --- pkg/controller/vacuum/controller.go | 26 +- pkg/controller/vacuum/controller_nil.go | 20 + pkg/controller/vacuum/mock.go | 27 + pkg/controller/vacuum/queue_store.go | 15 +- pkg/controller/vacuum/vacuum.go | 415 ++++++---------- pkg/controller/vacuum/vacuum_internal_test.go | 78 --- pkg/controller/vacuum/vacuum_test.go | 470 ++++++++---------- 7 files changed, 436 insertions(+), 615 deletions(-) create mode 100644 pkg/controller/vacuum/controller_nil.go create mode 100644 pkg/controller/vacuum/mock.go delete mode 100644 pkg/controller/vacuum/vacuum_internal_test.go diff --git a/pkg/controller/vacuum/controller.go b/pkg/controller/vacuum/controller.go index 67a82478c..06ad0454a 100644 --- a/pkg/controller/vacuum/controller.go +++ b/pkg/controller/vacuum/controller.go @@ -11,6 +11,7 @@ import ( "github.com/aquaproj/aqua/v2/pkg/config" "github.com/sirupsen/logrus" "github.com/spf13/afero" + "github.com/suzuki-shunsuke/logrus-error/logerr" bolt "go.etcd.io/bbolt" ) @@ -52,18 +53,18 @@ func (vc *Controller) getDB() (*bolt.DB, error) { Timeout: 1 * time.Second, }) if err != nil { - return nil, fmt.Errorf("failed to open database %v: %w", dbFile, err) + return nil, fmt.Errorf("open database %v: %w", dbFile, err) } if err := db.Update(func(tx *bolt.Tx) error { _, err := tx.CreateBucketIfNotExists([]byte(bucketNamePkgs)) if err != nil { - return fmt.Errorf("failed to create bucket '%v' in '%v': %w", bucketNamePkgs, dbFile, err) + return fmt.Errorf("create bucket '%v' in '%v': %w", bucketNamePkgs, dbFile, err) } return nil }); err != nil { db.Close() - return nil, fmt.Errorf("failed to create bucket: %w", err) + return nil, fmt.Errorf("create bucket: %w", err) } vc.db.Store(db) @@ -87,7 +88,7 @@ func (vc *Controller) withDBRetry(logE *logrus.Entry, fn func(*bolt.Tx) error, d logE.WithFields(logrus.Fields{ "attempt": i + 1, "error": err, - }).Warn("Retrying database operation") + }).Warn("retrying database operation") time.Sleep(backoff) backoff *= exponentialBackoff @@ -107,20 +108,20 @@ func (vc *Controller) withDB(logE *logrus.Entry, fn func(*bolt.Tx) error, dbAcce } defer func() { if err := vc.closeDB(); err != nil { - logE.WithError(err).Error("Failed to close database") + logerr.WithError(logE, err).Error("close database") } }() if dbAccessType == Update { err = db.Update(fn) if err != nil { - return fmt.Errorf("failed to view database: %w", err) + return fmt.Errorf("update database: %w", err) } return nil } err = db.View(fn) if err != nil { - return fmt.Errorf("failed to view database: %w", err) + return fmt.Errorf("view database: %w", err) } return nil } @@ -141,7 +142,7 @@ func (vc *Controller) closeDB() error { if vc.db.Load() != nil { if err := vc.db.Load().Close(); err != nil { - return fmt.Errorf("failed to close database: %w", err) + return fmt.Errorf("close database: %w", err) } vc.db.Store(nil) } @@ -150,8 +151,11 @@ func (vc *Controller) closeDB() error { } // Close closes the dependencies of the Controller. -func (vc *Controller) close(logE *logrus.Entry) error { - logE.Debug("Closing vacuum controller") +func (vc *Controller) Close(logE *logrus.Entry) error { + if !vc.IsVacuumEnabled(logE) { + return nil + } + logE.Debug("closing vacuum controller") if vc.storeQueue != nil { vc.storeQueue.close() } @@ -161,7 +165,7 @@ func (vc *Controller) close(logE *logrus.Entry) error { if vc.db.Load() != nil { if err := vc.db.Load().Close(); err != nil { - return fmt.Errorf("failed to close database: %w", err) + return fmt.Errorf("close database: %w", err) } vc.db.Store(nil) } diff --git a/pkg/controller/vacuum/controller_nil.go b/pkg/controller/vacuum/controller_nil.go new file mode 100644 index 000000000..373b01d2c --- /dev/null +++ b/pkg/controller/vacuum/controller_nil.go @@ -0,0 +1,20 @@ +package vacuum + +import ( + "github.com/aquaproj/aqua/v2/pkg/config" + "github.com/sirupsen/logrus" +) + +type NilVacuumController struct{} + +func (m *NilVacuumController) StorePackage(_ *logrus.Entry, _ *config.Package, _ string) error { + return nil +} + +func (m *NilVacuumController) Close(_ *logrus.Entry) error { + return nil +} + +func (m *NilVacuumController) Vacuum(_ *logrus.Entry) error { + return nil +} diff --git a/pkg/controller/vacuum/mock.go b/pkg/controller/vacuum/mock.go new file mode 100644 index 000000000..fdc9234f3 --- /dev/null +++ b/pkg/controller/vacuum/mock.go @@ -0,0 +1,27 @@ +package vacuum + +import ( + "github.com/aquaproj/aqua/v2/pkg/config" + "github.com/sirupsen/logrus" +) + +type MockVacuumController struct{} + +func NewMockVacuumController() *MockVacuumController { + return &MockVacuumController{} +} + +func (m *MockVacuumController) StorePackage(logE *logrus.Entry, pkg *config.Package, pkgPath string) error { + // Implementation of the mock method + return nil +} + +func (m *MockVacuumController) Close(logE *logrus.Entry) error { + // Implementation of the mock method + return nil +} + +func (m *MockVacuumController) Vacuum(logE *logrus.Entry) error { + // Implementation of the mock method + return nil +} diff --git a/pkg/controller/vacuum/queue_store.go b/pkg/controller/vacuum/queue_store.go index b69631472..f8e69b49b 100644 --- a/pkg/controller/vacuum/queue_store.go +++ b/pkg/controller/vacuum/queue_store.go @@ -4,12 +4,13 @@ import ( "sync" "github.com/sirupsen/logrus" + "github.com/suzuki-shunsuke/logrus-error/logerr" ) -// storeRequest represents a task to be processed. +// StoreRequest represents a task to be processed. type StoreRequest struct { logE *logrus.Entry - pkg []*ConfigPackage + pkg *Package } // StoreQueue manages a queue for handling tasks sequentially. @@ -44,7 +45,7 @@ func (sq *StoreQueue) worker() { } err := sq.vc.storePackageInternal(task.logE, task.pkg) if err != nil { - task.logE.WithField("vacuum", dbFile).WithError(err).Error("Failed to store package asynchronously") + logerr.WithError(task.logE, err).Error("store package asynchronously") } sq.wg.Done() case <-sq.done: @@ -53,7 +54,7 @@ func (sq *StoreQueue) worker() { task := <-sq.taskQueue err := sq.vc.storePackageInternal(task.logE, task.pkg) if err != nil { - task.logE.WithError(err).Error("Failed to store package asynchronously during shutdown") + logerr.WithError(task.logE, err).Error("store package asynchronously during shutdown") } sq.wg.Done() } @@ -62,8 +63,8 @@ func (sq *StoreQueue) worker() { } } -// Enqueue adds a task to the queue. -func (sq *StoreQueue) enqueue(logE *logrus.Entry, pkg []*ConfigPackage) { +// enqueue adds a task to the queue. +func (sq *StoreQueue) enqueue(logE *logrus.Entry, pkg *Package) { select { case <-sq.done: return @@ -76,7 +77,7 @@ func (sq *StoreQueue) enqueue(logE *logrus.Entry, pkg []*ConfigPackage) { } } -// Close waits for all tasks to complete and stops the worker. +// close waits for all tasks to complete and stops the worker. func (sq *StoreQueue) close() { sq.closeOnce.Do(func() { close(sq.done) diff --git a/pkg/controller/vacuum/vacuum.go b/pkg/controller/vacuum/vacuum.go index 6e88490d5..685e50443 100644 --- a/pkg/controller/vacuum/vacuum.go +++ b/pkg/controller/vacuum/vacuum.go @@ -1,12 +1,10 @@ package vacuum import ( - "context" "encoding/json" "errors" "fmt" "path/filepath" - "regexp" "strings" "sync" "time" @@ -16,6 +14,7 @@ import ( "github.com/ktr0731/go-fuzzyfinder" "github.com/sirupsen/logrus" "github.com/spf13/afero" + "github.com/suzuki-shunsuke/logrus-error/logerr" "go.etcd.io/bbolt" ) @@ -31,97 +30,38 @@ const ( type PackageVacuumEntries []PackageVacuumEntry type PackageVacuumEntry struct { - Key []byte + PkgPath []byte PackageEntry *PackageEntry } -type ConfigPackage struct { - Type string - Name string - Version string - PkgPath []string -} - type PackageEntry struct { LastUsageTime time.Time - PkgPath []string + Package *Package } -type Mode string - -const ( - ListPackages Mode = "list-packages" - ListExpiredPackages Mode = "list-expired-packages" - StorePackage Mode = "store-package" - StorePackages Mode = "store-packages" - AsyncStorePackage Mode = "async-store-package" - VacuumExpiredPackages Mode = "vacuum-expired-packages" - Close Mode = "close" -) - -// GetVacuumModeCLI returns the corresponding Mode based on the provided mode string. -// It supports the following modes: -// - "list-packages": returns ListPackages mode -// - "list-expired-packages": returns ListExpiredPackages mode -// - "vacuum-expired-packages": returns VacuumExpiredPackages mode -// If the provided mode string does not match any of the supported modes, it returns an error. -// -// Parameters: -// -// mode (string): The mode string to be converted to a Mode. -// -// Returns: -// -// (Mode, error): The corresponding Mode and nil if the mode string is valid, otherwise an empty Mode and an error. -func (vc *Controller) GetVacuumModeCLI(mode string) (Mode, error) { - switch mode { - case "list-packages": - return ListPackages, nil - case "list-expired-packages": - return ListExpiredPackages, nil - case "vacuum-expired-packages": - return VacuumExpiredPackages, nil - default: - return "", errors.New("invalid vacuum mode") - } +type Package struct { + Type string // Type of package (e.g. "github_release") + Name string // Name of package (e.g. "cli/cli") + Version string // Version of package (e.g. "v1.0.0") + PkgPath string // Path to the install path without the rootDir/pkgs/ prefix } -// Vacuum performs various vacuum operations based on the provided mode. -// Main function of vacuum controller. -func (vc *Controller) Vacuum(_ context.Context, logE *logrus.Entry, mode Mode, configPkg []*config.Package, args ...string) error { +// Vacuum performs the vacuuming process if it is enabled. +func (vc *Controller) Vacuum(logE *logrus.Entry) error { if !vc.IsVacuumEnabled(logE) { return nil } - - vacuumPkg := convertConfigPackages(configPkg) - - switch mode { - case ListPackages: - return vc.handleListPackages(logE, args...) - case ListExpiredPackages: - return vc.handleListExpiredPackages(logE, args...) - case StorePackage: - return vc.handleStorePackage(logE, vacuumPkg) - case StorePackages: - return vc.handleStorePackages(logE, vacuumPkg) - case AsyncStorePackage: - return vc.handleAsyncStorePackage(logE, vacuumPkg) - case VacuumExpiredPackages: - return vc.handleVacuumExpiredPackages(logE) - case Close: - return vc.close(logE) - } - - return errors.New("invalid vacuum mode") + return vc.vacuumExpiredPackages(logE) } -// convertConfigPackages converts a slice of config.Package pointers to a slice of ConfigPackage pointers. -func convertConfigPackages(configPkg []*config.Package) []*ConfigPackage { - vacuumPkg := make([]*ConfigPackage, 0, len(configPkg)) - for _, pkg := range configPkg { - vacuumPkg = append(vacuumPkg, vacuumConfigPackageFromConfigPackage(pkg)) +// ListPackages lists the packages based on the provided arguments. +// If the expired flag is set to true, it lists the expired packages. +// Otherwise, it lists all packages. +func (vc *Controller) ListPackages(logE *logrus.Entry, expired bool, args ...string) error { + if expired { + return vc.handleListExpiredPackages(logE, args...) } - return vacuumPkg + return vc.handleListPackages(logE, args...) } // handleListPackages retrieves a list of packages and displays them using a fuzzy search. @@ -143,50 +83,53 @@ func (vc *Controller) handleListExpiredPackages(logE *logrus.Entry, args ...stri return vc.displayPackagesFuzzy(logE, expiredPkgs, args...) } -// handleStorePackage processes a list of configuration packages and stores the first package in the list. -func (vc *Controller) handleStorePackage(logE *logrus.Entry, vacuumPkg []*ConfigPackage) error { - if len(vacuumPkg) < 1 { - return errors.New("StorePackage requires at least one configPackage") +// StorePackage stores the given package if vacuum is enabled. +// If the package is nil, it logs a warning and skips storing the package. +func (vc *Controller) StorePackage(logE *logrus.Entry, pkg *config.Package, pkgPath string) error { + if !vc.IsVacuumEnabled(logE) { + return nil } - defer func() { - if err := vc.close(logE); err != nil { - logE.WithError(err).Error("Failed to close vacuum DB after storing package") - } - }() - return vc.storePackageInternal(logE, []*ConfigPackage{vacuumPkg[0]}) + if pkg == nil { + logE.Warn("package is nil, skipping store package") + return nil + } + + vacuumPkg := vc.getVacuumPackage(pkg, pkgPath) + + return vc.handleAsyncStorePackage(logE, vacuumPkg) } -// handleStorePackages processes a list of configuration packages and stores them. -func (vc *Controller) handleStorePackages(logE *logrus.Entry, vacuumPkg []*ConfigPackage) error { - if len(vacuumPkg) < 1 { - return errors.New("StorePackages requires at least one configPackage") +// getVacuumPackage converts a config +func (vc *Controller) getVacuumPackage(configPkg *config.Package, pkgPath string) *Package { + pkgPath = generatePackageKey(vc.Param.RootDir, pkgPath) + return &Package{ + Type: configPkg.PackageInfo.Type, + Name: configPkg.Package.Name, + Version: configPkg.Package.Version, + PkgPath: pkgPath, } - defer func() { - if err := vc.close(logE); err != nil { - logE.WithError(err).Error("Failed to close vacuum DB after storing multiple packages") - } - }() - return vc.storePackageInternal(logE, vacuumPkg) +} + +// generatePackageKey generates a package key based on the root directory and package path. +func generatePackageKey(rootDir string, pkgPath string) string { + const splitParts = 2 + pkgPath = strings.SplitN(pkgPath, rootDir+"/pkgs/", splitParts)[1] + return pkgPath } // handleAsyncStorePackage processes a list of configuration packages asynchronously. -func (vc *Controller) handleAsyncStorePackage(logE *logrus.Entry, vacuumPkg []*ConfigPackage) error { - if len(vacuumPkg) < 1 { - return errors.New("AsyncStorePackage requires at least one configPackage") +func (vc *Controller) handleAsyncStorePackage(logE *logrus.Entry, vacuumPkg *Package) error { + if vacuumPkg == nil { + return errors.New("vacuumPkg is nil") } vc.storeQueue.enqueue(logE, vacuumPkg) return nil } -// handleVacuumExpiredPackages handles the process of vacuuming expired packages. -func (vc *Controller) handleVacuumExpiredPackages(logE *logrus.Entry) error { - return vc.vacuumExpiredPackages(logE) -} - // IsVacuumEnabled checks if the vacuum feature is enabled based on the configuration. func (vc *Controller) IsVacuumEnabled(logE *logrus.Entry) bool { - if vc.Param.VacuumDays == nil || *vc.Param.VacuumDays <= 0 { - logE.Debug("Vacuum is disabled. AQUA_VACUUM_DAYS is not set or invalid.") + if vc.Param.VacuumDays <= 0 { + logE.Debug("vacuum is disabled. AQUA_VACUUM_DAYS is not set or invalid.") return false } return true @@ -211,8 +154,15 @@ func (vc *Controller) listExpiredPackages(logE *logrus.Entry) ([]*PackageVacuumE // isPackageExpired checks if a package is expired based on the vacuum configuration. func (vc *Controller) isPackageExpired(pkg *PackageVacuumEntry) bool { const secondsInADay = 24 * 60 * 60 - threshold := int64(*vc.Param.VacuumDays) * secondsInADay - return time.Since(pkg.PackageEntry.LastUsageTime).Seconds() > float64(threshold) + threshold := vc.Param.VacuumDays * secondsInADay + + lastUsageTime := pkg.PackageEntry.LastUsageTime + if lastUsageTime.Location() != time.UTC { + lastUsageTime = lastUsageTime.In(time.UTC) + } + + timeSinceLastUsage := time.Since(lastUsageTime).Seconds() + return timeSinceLastUsage > float64(threshold) } // listPackages lists all stored package entries. @@ -237,12 +187,12 @@ func (vc *Controller) listPackages(logE *logrus.Entry) ([]*PackageVacuumEntry, e pkgEntry, err := decodePackageEntry(value) if err != nil { logE.WithFields(logrus.Fields{ - "pkgKey": string(k), - }).Warnf("Failed to decode entry: %v", err) + "pkg_key": string(k), + }).Warnf("unable to decode entry: %v", err) return err } pkgs = append(pkgs, &PackageVacuumEntry{ - Key: append([]byte{}, k...), + PkgPath: append([]byte{}, k...), PackageEntry: pkgEntry, }) return nil @@ -272,7 +222,7 @@ func (vc *Controller) displayPackagesFuzzyTest(logE *logrus.Entry, pkgs []*Packa func (vc *Controller) displayPackagesFuzzy(logE *logrus.Entry, pkgs []*PackageVacuumEntry, args ...string) error { if len(pkgs) == 0 { - logE.Info("No packages to display") + logE.Info("no packages to display") return nil } if len(args) > 0 && args[0] == "test" { @@ -290,7 +240,7 @@ func (vc *Controller) displayPackagesFuzzyInteractive(pkgs []*PackageVacuumEntry return fmt.Sprintf("%s%s [%s]", expiredString, - pkgs[i].Key, + pkgs[i].PkgPath, humanize.Time(pkgs[i].PackageEntry.LastUsageTime), ) }, @@ -303,10 +253,6 @@ func (vc *Controller) displayPackagesFuzzyInteractive(pkgs []*PackageVacuumEntry if vc.isPackageExpired(pkg) { expiredString = "Expired ⌛" } - parsedConfigPkg, err := generateConfigPackageFromKey(pkg.Key) - if err != nil { - return fmt.Sprintf("Failed to parse package key: %v", err) - } return fmt.Sprintf( "Package Details:\n\n"+ "%s \n"+ @@ -314,15 +260,13 @@ func (vc *Controller) displayPackagesFuzzyInteractive(pkgs []*PackageVacuumEntry "Package: %s\n"+ "Version: %s\n\n"+ "Last Used: %s\n"+ - "Last Used (exact): %s\n\n"+ - "PkgPath: %v\n", + "Last Used (exact): %s\n\n", expiredString, - parsedConfigPkg.Type, - parsedConfigPkg.Name, - parsedConfigPkg.Version, + pkg.PackageEntry.Package.Type, + pkg.PackageEntry.Package.Name, + pkg.PackageEntry.Package.Version, humanize.Time(pkg.PackageEntry.LastUsageTime), - pkg.PackageEntry.LastUsageTime.Format("2024-12-31 15:04:05"), - pkg.PackageEntry.PkgPath, + pkg.PackageEntry.LastUsageTime.Format("2006-01-02 15:04:05"), ) }), fuzzyfinder.WithHeader("Navigate through packages to display details"), @@ -339,22 +283,25 @@ func (vc *Controller) displayPackagesFuzzyInteractive(pkgs []*PackageVacuumEntry // vacuumExpiredPackages performs cleanup of expired packages. func (vc *Controller) vacuumExpiredPackages(logE *logrus.Entry) error { - expired, err := vc.listExpiredPackages(logE) + expiredPackages, err := vc.listExpiredPackages(logE) if err != nil { return err } - if len(expired) == 0 { + if len(expiredPackages) == 0 { + logE.Info("no expired packages to remove") return nil } - successKeys, errCh := vc.processExpiredPackages(logE, expired) + successfulRemovals, errorsEncountered := vc.processExpiredPackages(logE, expiredPackages) - if len(errCh) > 0 { + if len(errorsEncountered) > 0 { return errors.New("some packages could not be removed") } - if len(successKeys) > 0 { - if err := vc.removePackages(logE, successKeys); err != nil { + + defer vc.Close(logE) + if len(successfulRemovals) > 0 { + if err := vc.removePackages(logE, successfulRemovals); err != nil { return fmt.Errorf("failed to remove packages from database: %w", err) } } @@ -370,9 +317,9 @@ func (vc *Controller) vacuumExpiredPackages(logE *logrus.Entry) error { // - expired: A slice of PackageVacuumEntry representing the expired packages to be processed. // // Returns: -// - A slice of ConfigPackage representing the packages that were successfully processed and need to be removed from the vacuum database. +// - A slice of VacuumPackage representing the packages that were successfully processed and need to be removed from the vacuum database. // - A slice of errors encountered during the processing of the expired packages. -func (vc *Controller) processExpiredPackages(logE *logrus.Entry, expired []*PackageVacuumEntry) ([]*ConfigPackage, []error) { //nolint:funlen +func (vc *Controller) processExpiredPackages(logE *logrus.Entry, expired []*PackageVacuumEntry) ([]string, []error) { const batchSize = 10 successKeys := make(chan string, len(expired)) errCh := make(chan error, len(expired)) @@ -385,34 +332,35 @@ func (vc *Controller) processExpiredPackages(logE *logrus.Entry, expired []*Pack } batch := make([]struct { - key string - pkgPath []string + pkgPath string + pkgType string + pkgName string version string }, len(expired[i:end])) for j, entry := range expired[i:end] { - batch[j].key = string(entry.Key) - batch[j].pkgPath = entry.PackageEntry.PkgPath - batch[j].version = strings.Split(string(entry.Key), "@")[1] + batch[j].pkgPath = string(entry.PkgPath) + batch[j].pkgType = entry.PackageEntry.Package.Type + batch[j].pkgName = entry.PackageEntry.Package.Name + batch[j].version = entry.PackageEntry.Package.Version } wg.Add(1) go func(batch []struct { - key string - pkgPath []string + pkgPath string + pkgType string + pkgName string version string }, ) { defer wg.Done() for _, entry := range batch { - for _, path := range entry.pkgPath { - if err := vc.removePackageVersionPath(vc.Param, path, entry.version); err != nil { - logE.WithField("expiredPackages", entry.key).WithError(err).Error("Error removing path") - errCh <- err - continue - } - successKeys <- entry.key + if err := vc.removePackageVersionPath(vc.Param, entry.pkgPath, entry.pkgType); err != nil { + logerr.WithError(logE, err).Error("removing path") + errCh <- err + continue } + successKeys <- entry.pkgPath } }(batch) } @@ -421,14 +369,9 @@ func (vc *Controller) processExpiredPackages(logE *logrus.Entry, expired []*Pack close(successKeys) close(errCh) - ConfigPackageToRemove := make([]*ConfigPackage, 0, len(expired)) - for key := range successKeys { - pkg, err := generateConfigPackageFromKey([]byte(key)) - if err != nil { - logE.WithField("key", key).WithError(err).Error("Failed to generate package from key") - continue - } - ConfigPackageToRemove = append(ConfigPackageToRemove, pkg) + pathsToRemove := make([]string, 0, len(expired)) + for pkgPath := range successKeys { + pathsToRemove = append(pathsToRemove, pkgPath) } errors := make([]error, 0, len(expired)) @@ -436,11 +379,11 @@ func (vc *Controller) processExpiredPackages(logE *logrus.Entry, expired []*Pack errors = append(errors, err) } - return ConfigPackageToRemove, errors + return pathsToRemove, errors } // storePackageInternal stores package entries in the database. -func (vc *Controller) storePackageInternal(logE *logrus.Entry, pkgs []*ConfigPackage, dateTime ...time.Time) error { +func (vc *Controller) storePackageInternal(logE *logrus.Entry, pkg *Package, dateTime ...time.Time) error { lastUsedTime := time.Now() if len(dateTime) > 0 { lastUsedTime = dateTime[0] @@ -450,73 +393,77 @@ func (vc *Controller) storePackageInternal(logE *logrus.Entry, pkgs []*ConfigPac if b == nil { return errors.New("bucket not found") } + logE.WithFields(logrus.Fields{ + "name": pkg.Name, + "version": pkg.Version, + "pkg_path": pkg.PkgPath, + }).Debug("storing package in vacuum database") + + pkgKey := pkg.PkgPath + pkgEntry := &PackageEntry{ + LastUsageTime: lastUsedTime, + Package: pkg, + } - for _, pkg := range pkgs { - logE.WithFields(logrus.Fields{ - "name": pkg.Name, - "version": pkg.Version, - "PkgPath": pkg.PkgPath, - }).Debug("Storing package in vacuum database") - pkgEntry := &PackageEntry{ - LastUsageTime: lastUsedTime, - PkgPath: pkg.PkgPath, - } - - data, err := encodePackageEntry(pkgEntry) - if err != nil { - logE.WithFields(logrus.Fields{ - "name": pkg.Name, - "version": pkg.Version, - }).WithError(err).Error("Failed to encode package") - return fmt.Errorf("encode package %s: %w", pkg.Name, err) - } + data, err := encodePackageEntry(pkgEntry) + if err != nil { + logerr.WithError(logE, err).WithFields( + logrus.Fields{ + "name": pkg.Name, + "version": pkg.Version, + "pkg_path": pkg.PkgPath, + }).Error("encode package") + return fmt.Errorf("encode package %s: %w", pkg.Name, err) + } - if err := b.Put(generateKey(pkg), data); err != nil { - logE.WithField("pkgKey", pkg).WithError(err).Error("Failed to store package in vacuum database") - return fmt.Errorf("store package %s: %w", pkg.Name, err) - } + if err := b.Put([]byte(pkgKey), data); err != nil { + logerr.WithError(logE, err).WithField("pkgKey", pkgKey).Error("store package in vacuum database") + return fmt.Errorf("store package %s: %w", pkg.Name, err) } return nil }, Update) } // removePackages removes package entries from the database. -func (vc *Controller) removePackages(logE *logrus.Entry, pkgs []*ConfigPackage) error { - keys := make([]string, 0, len(pkgs)) - for _, pkg := range pkgs { - keys = append(keys, string(generateKey(pkg))) - } +func (vc *Controller) removePackages(logE *logrus.Entry, pkgs []string) error { return vc.withDBRetry(logE, func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(bucketNamePkgs)) if b == nil { return errors.New("bucket not found") } - for _, key := range keys { + for _, key := range pkgs { if err := b.Delete([]byte(key)); err != nil { return fmt.Errorf("delete package %s: %w", key, err) } + logE.WithField("pkgKey", key).Info("removed package from vacuum database") } return nil }, Update) } // removePackageVersionPath removes the specified package version directory and its parent directory if it becomes empty. -func (vc *Controller) removePackageVersionPath(param *config.Param, path string, version string) error { - pkgVersionPath := filepath.Join(param.RootDir, "pkgs", path, version) +func (vc *Controller) removePackageVersionPath(param *config.Param, path string, pkgType string) error { + pkgsPath := filepath.Join(param.RootDir, "pkgs") + pkgsRemoveLimitPath := filepath.Join(pkgsPath, pkgType) + pkgVersionPath := filepath.Join(pkgsPath, path) if err := vc.fs.RemoveAll(pkgVersionPath); err != nil { return fmt.Errorf("remove package version directories: %w", err) } - pkgPath := filepath.Join(param.RootDir, "pkgs", path) - dirIsEmpty, err := afero.IsEmpty(vc.fs, pkgPath) - if err != nil { - return fmt.Errorf("check if the directory is empty: %w", err) - } - if dirIsEmpty { - if err := vc.fs.RemoveAll(pkgPath); err != nil { + currentPath := filepath.Dir(pkgVersionPath) + for currentPath != pkgsRemoveLimitPath { + dirIsEmpty, err := afero.IsEmpty(vc.fs, currentPath) + if err != nil { + return fmt.Errorf("check if directory is empty: %w", err) + } + if !dirIsEmpty { + break + } + if err := vc.fs.RemoveAll(currentPath); err != nil { return fmt.Errorf("remove package directories: %w", err) } + currentPath = filepath.Dir(currentPath) } return nil } @@ -539,15 +486,32 @@ func decodePackageEntry(data []byte) (*PackageEntry, error) { return &pkgEntry, nil } -// retrievePackageEntry retrieves a package entry from the database by key. -func (vc *Controller) retrievePackageEntry(logE *logrus.Entry, key []byte) (*PackageEntry, error) { +// GetPackageLastUsed retrieves the last used time of a package. for testing purposes. +func (vc *Controller) GetPackageLastUsed(logE *logrus.Entry, pkgPath string) *time.Time { + var lastUsedTime time.Time + pkgEntry, _ := vc.retrievePackageEntry(logE, pkgPath) + if pkgEntry != nil { + lastUsedTime = pkgEntry.LastUsageTime + } + return &lastUsedTime +} + +// SetTimeStampPackage permit define a Timestamp for a package Manually. for testing purposes. +func (vc *Controller) SetTimestampPackage(logE *logrus.Entry, pkg *config.Package, pkgPath string, datetime time.Time) error { + vacuumPkg := vc.getVacuumPackage(pkg, pkgPath) + return vc.storePackageInternal(logE, vacuumPkg, datetime) +} + +// retrievePackageEntry retrieves a package entry from the database by key. for testing purposes. +func (vc *Controller) retrievePackageEntry(logE *logrus.Entry, key string) (*PackageEntry, error) { var pkgEntry *PackageEntry + key = generatePackageKey(vc.Param.RootDir, key) err := vc.withDBRetry(logE, func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(bucketNamePkgs)) if b == nil { return nil } - value := b.Get(key) + value := b.Get([]byte(key)) if value == nil { return nil } @@ -558,56 +522,3 @@ func (vc *Controller) retrievePackageEntry(logE *logrus.Entry, key []byte) (*Pac }, View) return pkgEntry, err } - -// generateKey generates a unique key for a package. -func generateKey(pkg *ConfigPackage) []byte { - return []byte(pkg.Type + "," + pkg.Name + "@" + pkg.Version) -} - -// vacuumConfigPackageFromConfigPackage returns a ConfigPackage config from a config.Package. -func vacuumConfigPackageFromConfigPackage(pkg *config.Package) *ConfigPackage { - pkgPathsMap := pkg.PackageInfo.PkgPaths() - PkgPath := make([]string, 0, len(pkgPathsMap)) - for k := range pkgPathsMap { - PkgPath = append(PkgPath, k) - } - - return &ConfigPackage{ - Type: pkg.PackageInfo.Type, - Name: pkg.Package.Name, - Version: pkg.Package.Version, - PkgPath: PkgPath, - } -} - -// generateConfigPackageFromKey return a minimal package config from a key. -func generateConfigPackageFromKey(key []byte) (*ConfigPackage, error) { - if len(key) == 0 { - return nil, errors.New("empty key") - } - pattern := `^(?P[^,]+),(?P[^@]+)@(?P.+)$` - re := regexp.MustCompile(pattern) - match := re.FindStringSubmatch(string(key)) - - if match == nil { - return nil, fmt.Errorf("key %s does not match the pattern %s", key, pattern) - } - - result := make(map[string]string) - for i, name := range re.SubexpNames() { - if i != 0 && name != "" { - result[name] = match[i] - } - } - - packageInfoType := result["PackageInfo_Type"] - packageName := result["Package_Name"] - packageVersion := result["Package_Version"] - - pkg := &ConfigPackage{ - Type: packageInfoType, - Name: packageName, - Version: packageVersion, - } - return pkg, nil -} diff --git a/pkg/controller/vacuum/vacuum_internal_test.go b/pkg/controller/vacuum/vacuum_internal_test.go deleted file mode 100644 index c1fb460ce..000000000 --- a/pkg/controller/vacuum/vacuum_internal_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package vacuum - -import ( - "testing" - "time" - - "github.com/aquaproj/aqua/v2/pkg/config" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestGenerateConfigPackageFromKey(t *testing.T) { - t.Parallel() - testCases := []struct { - name string - key string - expected *ConfigPackage - expectError bool - }{ - { - name: "Valid key", - key: "github_release,test/pkg@v1.0.0", - expected: &ConfigPackage{ - Type: "github_release", - Name: "test/pkg", - Version: "v1.0.0", - }, - expectError: false, - }, - { - name: "Invalid key format", - key: "invalid_key_format", - expected: nil, - expectError: true, - }, - { - name: "Empty key", - key: "", - expected: nil, - expectError: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - result, err := generateConfigPackageFromKey([]byte(tc.key)) - if tc.expectError { - require.Error(t, err) - } else { - require.NoError(t, err) - assert.Equal(t, tc.expected, result) - } - }) - } -} - -// GetPackageLastUsed retrieves the last used time of a package. for testing purposes. -func (vc *Controller) GetPackageLastUsed(logE *logrus.Entry, pkg *config.Package) *time.Time { - var lastUsedTime time.Time - vacuumpkg := vacuumConfigPackageFromConfigPackage(pkg) - key := generateKey(vacuumpkg) - pkgEntry, _ := vc.retrievePackageEntry(logE, key) - if pkgEntry != nil { - lastUsedTime = pkgEntry.LastUsageTime - } - return &lastUsedTime -} - -// SetTimeStampPackage permit define a Timestamp for a package Manually. for testing purposes. -func (vc *Controller) SetTimestampPackages(logE *logrus.Entry, pkg []*config.Package, datetime time.Time) error { - vacuumPkgs := make([]*ConfigPackage, 0, len(pkg)) - for _, p := range pkg { - vacuumPkgs = append(vacuumPkgs, vacuumConfigPackageFromConfigPackage(p)) - } - return vc.storePackageInternal(logE, vacuumPkgs, datetime) -} diff --git a/pkg/controller/vacuum/vacuum_test.go b/pkg/controller/vacuum/vacuum_test.go index 823490ce6..439f78376 100644 --- a/pkg/controller/vacuum/vacuum_test.go +++ b/pkg/controller/vacuum/vacuum_test.go @@ -1,9 +1,7 @@ package vacuum_test import ( - "context" "path/filepath" - "runtime" "testing" "time" @@ -28,13 +26,12 @@ const ( Close string = "close" ) -func TestVacuum(t *testing.T) { //nolint:funlen,maintidx +func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop t.Parallel() // Setup common test fixtures logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) - ctx := context.Background() fs := afero.NewOsFs() // Create temp directory for tests @@ -53,36 +50,29 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx require.NoError(t, err) // Setup param := &config.Param{ - VacuumDays: nil, - RootDir: testDir, + RootDir: testDir, } controller := vacuum.New(param, fs) - // Test - err = controller.Vacuum(ctx, logE, vacuum.Mode(ListPackages), nil) - - // Assert + err = controller.ListPackages(logE, false, "test") require.NoError(t, err, "Should return nil when vacuum is disabled") }) - t.Run("invalid mode", func(t *testing.T) { + t.Run("vacuum bad configuration", func(t *testing.T) { t.Parallel() - testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_invalid_mode") + logE.Logger.Level = logrus.DebugLevel + testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_bad_config") require.NoError(t, err) // Setup - days := 30 param := &config.Param{ - VacuumDays: &days, RootDir: testDir, + VacuumDays: -1, } controller := vacuum.New(param, fs) - // Test - err = controller.Vacuum(ctx, logE, vacuum.Mode("invalid"), nil) - - // Assert - require.Error(t, err) - assert.Contains(t, err.Error(), "invalid vacuum mode") + err = controller.StorePackage(logE, nil, testDir) + require.NoError(t, err, "Should return nil when vacuum is disabled") + assert.Equal(t, "vacuum is disabled. AQUA_VACUUM_DAYS is not set or invalid.", hook.LastEntry().Message) }) t.Run("ListPackages mode - empty database", func(t *testing.T) { @@ -93,33 +83,33 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx days := 30 param := &config.Param{ - VacuumDays: &days, + VacuumDays: days, RootDir: testDir, } controller := vacuum.New(param, fs) // Test - err = controller.Vacuum(ctx, logE, vacuum.Mode(ListPackages), nil, "test") + err = controller.ListPackages(logE, false, "test") // Assert require.NoError(t, err) // Should succeed with empty database - assert.Equal(t, "No packages to display", hook.LastEntry().Message) + assert.Equal(t, "no packages to display", hook.LastEntry().Message) }) - t.Run("AsyncFailed", func(t *testing.T) { + t.Run("StoreFailed", func(t *testing.T) { t.Parallel() - testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_async_failed") + testDir, err := afero.TempDir(fs, tempTestDir, "store_failed") require.NoError(t, err) days := 1 // Short expiration for testing param := &config.Param{ - VacuumDays: &days, + VacuumDays: days, RootDir: testDir, } controller := vacuum.New(param, fs) - numberPackagesToStore := 4 - pkgs := generateTestPackages(numberPackagesToStore) + numberPackagesToStore := 7 + pkgs := generateTestPackages(numberPackagesToStore, param.RootDir) // We force Keeping the DB open to simulate a failure in the async operation if err := controller.TestKeepDBOpen(); err != nil { @@ -127,18 +117,18 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx } hook.Reset() for _, pkg := range pkgs { - err := controller.Vacuum(context.Background(), logE, vacuum.Mode(AsyncStorePackage), []*config.Package{pkg}) + err := controller.StorePackage(logE, pkg.configPkg, pkg.pkgPath) require.NoError(t, err) } // Wait for the async operations to complete - err = controller.Vacuum(context.Background(), logE, vacuum.Mode(Close), nil) + err = controller.Close(logE) require.NoError(t, err) // If AsyncStorePackage fails, Close should wait for the async operations to complete, but not return an error expectedLogMessage := []string{ - "Failed to store package asynchronously", - "Retrying database operation", - "Failed to store package asynchronously during shutdown", + "store package asynchronously", + "retrying database operation", + "store package asynchronously during shutdown", } var receivedMessages []string for _, entry := range hook.AllEntries() { @@ -157,20 +147,23 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx days := 30 param := &config.Param{ - VacuumDays: &days, + VacuumDays: days, RootDir: testDir, } controller := vacuum.New(param, fs) numberPackagesToStore := 1 - pkgs := generateTestPackages(numberPackagesToStore) + pkgs := generateTestPackages(numberPackagesToStore, param.RootDir) // Store the package - err = controller.Vacuum(ctx, logE, vacuum.Mode(StorePackage), pkgs) + err = controller.StorePackage(logE, pkgs[0].configPkg, pkgs[0].pkgPath) + require.NoError(t, err) + + err = controller.Close(logE) // Close to ensure async operations are completed require.NoError(t, err) // List packages - should contain our stored package - err = controller.Vacuum(ctx, logE, vacuum.Mode(ListPackages), nil, "test") + err = controller.ListPackages(logE, false, "test") require.NoError(t, err) assert.Equal(t, "Test mode: Displaying packages", hook.LastEntry().Message) assert.Equal(t, 1, hook.LastEntry().Data["TotalPackages"]) @@ -178,31 +171,36 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx hook.Reset() // Verify package was stored correctly - lastUsed := controller.GetPackageLastUsed(logE, pkgs[0]) + lastUsed := controller.GetPackageLastUsed(logE, pkgs[0].pkgPath) assert.False(t, lastUsed.IsZero(), "Package should have a last used time") }) - t.Run("StorePackages", func(t *testing.T) { + t.Run("StoreMultiplePackages", func(t *testing.T) { t.Parallel() - testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_storePackages_test") + testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_StoreMultiplePackages_test") require.NoError(t, err) days := 30 param := &config.Param{ - VacuumDays: &days, + VacuumDays: days, RootDir: testDir, } controller := vacuum.New(param, fs) numberPackagesToStore := 4 - pkgs := generateTestPackages(numberPackagesToStore) + pkgs := generateTestPackages(numberPackagesToStore, param.RootDir) // Store the package - err = controller.Vacuum(ctx, logE, vacuum.Mode(StorePackages), pkgs) + for _, pkg := range pkgs { + err := controller.StorePackage(logE, pkg.configPkg, pkg.pkgPath) + require.NoError(t, err) + } + + err = controller.Close(logE) // Close to ensure async operations are completed require.NoError(t, err) // List packages - should contain our stored package - err = controller.Vacuum(ctx, logE, vacuum.Mode(ListPackages), nil, "test") + err = controller.ListPackages(logE, false, "test") require.NoError(t, err) assert.Equal(t, "Test mode: Displaying packages", hook.LastEntry().Message) assert.Equal(t, 4, hook.LastEntry().Data["TotalPackages"]) @@ -210,43 +208,22 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx hook.Reset() }) - t.Run("GetVacuumModeCLI valid modes", func(t *testing.T) { + t.Run("StoreNilPackage", func(t *testing.T) { t.Parallel() - testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_getVacuumModeCLI_test") + testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_StoreNilPackage_test") require.NoError(t, err) - param := &config.Param{ - RootDir: testDir, - } - controller := vacuum.New(param, fs) - // Test valid modes - modes := map[string]vacuum.Mode{ - "list-packages": vacuum.ListPackages, - "list-expired-packages": vacuum.ListExpiredPackages, - "vacuum-expired-packages": vacuum.VacuumExpiredPackages, - } - - for modeStr, expectedMode := range modes { - mode, err := controller.GetVacuumModeCLI(modeStr) - require.NoError(t, err) - assert.Equal(t, expectedMode, mode) - } - }) - - t.Run("GetVacuumModeCLI invalid mode", func(t *testing.T) { - t.Parallel() - testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_getVacuumModeCLI_invalid_test") - require.NoError(t, err) + days := 30 param := &config.Param{ - RootDir: testDir, + VacuumDays: days, + RootDir: testDir, } controller := vacuum.New(param, fs) - // Test invalid mode - mode, err := controller.GetVacuumModeCLI("invalid-mode") - require.Error(t, err) - assert.Equal(t, vacuum.Mode(""), mode) - assert.Contains(t, err.Error(), "invalid vacuum mode") + // Store the package + err = controller.StorePackage(logE, nil, tempTestDir) + require.NoError(t, err) + assert.Equal(t, "package is nil, skipping store package", hook.LastEntry().Message) }) t.Run("handleListExpiredPackages - no expired packages", func(t *testing.T) { @@ -256,76 +233,18 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx days := 30 param := &config.Param{ - VacuumDays: &days, + VacuumDays: days, RootDir: testDir, } controller := vacuum.New(param, fs) // Test - err = controller.Vacuum(ctx, logE, vacuum.Mode(ListExpiredPackages), nil, "test") + err = controller.ListPackages(logE, true, "test") // Assert require.NoError(t, err) // Error if no package found - assert.Equal(t, "No packages to display", hook.LastEntry().Message) - }) - - t.Run("handleStorePackage - invalid package count", func(t *testing.T) { - t.Parallel() - testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_handle_store_package") - require.NoError(t, err) - days := 30 - param := &config.Param{ - VacuumDays: &days, - RootDir: testDir, - } - controller := vacuum.New(param, fs) - - // Test with no packages - err = controller.Vacuum(ctx, logE, vacuum.Mode(StorePackage), nil) - - // Assert - require.Error(t, err) - assert.Contains(t, err.Error(), "StorePackage requires at least one configPackage") + assert.Equal(t, "no packages to display", hook.LastEntry().Message) }) - - t.Run("handleStorePackages - invalid package count", func(t *testing.T) { - t.Parallel() - testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_handle_store_packages") - require.NoError(t, err) - days := 30 - param := &config.Param{ - VacuumDays: &days, - RootDir: testDir, - } - controller := vacuum.New(param, fs) - - // Test with no packages - err = controller.Vacuum(ctx, logE, vacuum.Mode(StorePackages), nil) - - // Assert - require.Error(t, err) - assert.Contains(t, err.Error(), "StorePackages requires at least one configPackage") - }) - - t.Run("handleAsyncStorePackage - invalid package count", func(t *testing.T) { - t.Parallel() - testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_handle_async_store_package") - require.NoError(t, err) - days := 30 - param := &config.Param{ - VacuumDays: &days, - RootDir: testDir, - } - controller := vacuum.New(param, fs) - - // Test with no packages - err = controller.Vacuum(ctx, logE, vacuum.Mode(AsyncStorePackage), nil) - - // Assert - require.Error(t, err) - assert.Contains(t, err.Error(), "AsyncStorePackage requires at least one configPackage") - }) - t.Run("VacuumExpiredPackages workflow", func(t *testing.T) { t.Parallel() testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_expire_test") @@ -337,64 +256,66 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx days := 1 // Short expiration for testing param := &config.Param{ - VacuumDays: &days, + VacuumDays: days, RootDir: testDir, } controller := vacuum.New(param, fs) numberPackagesToStore := 3 numberPackagesToExpire := 1 - pkgs := generateTestPackages(numberPackagesToStore) - pkgPaths := make([]string, 0, len(pkgs)) + pkgs := generateTestPackages(numberPackagesToStore, param.RootDir) + pkgPaths := make([]string, 0, numberPackagesToStore) // Create package paths and files for _, pkg := range pkgs { - pkgPath := filepath.Join(testDir, "pkgs", "github_release/github.com/"+pkg.Package.Name, pkg.Package.Version) - err = fs.MkdirAll(pkgPath, 0o755) + err = fs.MkdirAll(pkg.pkgPath, 0o755) require.NoError(t, err) // Create a test file in the package directory - testFile := filepath.Join(pkgPath, "test.txt") + testFile := filepath.Join(pkg.pkgPath, "test.txt") err = afero.WriteFile(fs, testFile, []byte("test content"), 0o644) require.NoError(t, err) - pkgPaths = append(pkgPaths, pkgPath) + pkgPaths = append(pkgPaths, pkg.pkgPath) } // Store Multiple packages - err = controller.Vacuum(ctx, logE, vacuum.Mode(AsyncStorePackage), pkgs) - require.NoError(t, err) + for _, pkg := range pkgs { + err = controller.StorePackage(logE, pkg.configPkg, pkg.pkgPath) + require.NoError(t, err) + } // Call Close to ensure all async operations are completed - err = controller.Vacuum(ctx, logE, vacuum.Mode(Close), nil) + err = controller.Close(logE) require.NoError(t, err) // Modify timestamp of one package to be expired oldTime := time.Now().Add(-48 * time.Hour) // 2 days old - err = controller.SetTimestampPackages(logE, pkgs[0:numberPackagesToExpire], oldTime) - require.NoError(t, err) + for _, pkg := range pkgs[:numberPackagesToExpire] { + err = controller.SetTimestampPackage(logE, pkg.configPkg, pkg.pkgPath, oldTime) + require.NoError(t, err) + } // Check Packages after expiration - err = controller.Vacuum(ctx, logE, vacuum.Mode(ListPackages), nil, "test") + err = controller.ListPackages(logE, false, "test") require.NoError(t, err) assert.Equal(t, numberPackagesToStore, hook.LastEntry().Data["TotalPackages"]) assert.Equal(t, numberPackagesToExpire, hook.LastEntry().Data["TotalExpired"]) // List expired packages only - err = controller.Vacuum(ctx, logE, vacuum.Mode(ListExpiredPackages), nil, "test") + err = controller.ListPackages(logE, true, "test") require.NoError(t, err) assert.Equal(t, numberPackagesToExpire, hook.LastEntry().Data["TotalPackages"]) assert.Equal(t, numberPackagesToExpire, hook.LastEntry().Data["TotalExpired"]) // Run vacuum - err = controller.Vacuum(ctx, logE, vacuum.Mode(VacuumExpiredPackages), nil) + err = controller.Vacuum(logE) require.NoError(t, err) // List expired packages - err = controller.Vacuum(ctx, logE, vacuum.Mode(ListPackages), nil, "test") + err = controller.ListPackages(logE, true, "test") require.NoError(t, err) - assert.Equal(t, numberPackagesToStore-numberPackagesToExpire, hook.LastEntry().Data["TotalPackages"]) - assert.Equal(t, 0, hook.LastEntry().Data["TotalExpired"]) + assert.Equal(t, "no packages to display", hook.LastEntry().Message) // Verify Package Paths was removed : for _, pkgPath := range pkgPaths[:numberPackagesToExpire] { @@ -404,62 +325,152 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx } // Modify timestamp of one package to be expired And lock DB to simulate a failure in the vacuum operation - err = controller.SetTimestampPackages(logE, pkgs[numberPackagesToExpire:], oldTime) - require.NoError(t, err) + for _, pkg := range pkgs[:numberPackagesToExpire] { + err = controller.SetTimestampPackage(logE, pkg.configPkg, pkg.pkgPath, oldTime) + require.NoError(t, err) + } // Keep Database open to simulate a failure in the vacuum operation err = controller.TestKeepDBOpen() require.NoError(t, err) // Run vacuum - err = controller.Vacuum(ctx, logE, vacuum.Mode(VacuumExpiredPackages), nil) + err = controller.Vacuum(logE) require.Error(t, err) - assert.Contains(t, err.Error(), "failed to open database vacuum.db: timeout") + assert.Contains(t, err.Error(), "open database vacuum.db: timeout") }) t.Run("TestVacuumWithoutExpiredPackages", func(t *testing.T) { t.Parallel() - testDir, err := afero.TempDir(afero.NewOsFs(), "", "vacuum_no_expired") + fs := afero.NewOsFs() + testDir, err := afero.TempDir(fs, "", "vacuum_no_expired") require.NoError(t, err) days := 30 param := &config.Param{ - VacuumDays: &days, + VacuumDays: days, RootDir: testDir, } - controller := vacuum.New(param, afero.NewOsFs()) + controller := vacuum.New(param, fs) // Store non-expired packages - pkgs := generateTestPackages(3) - err = controller.Vacuum(ctx, logE, vacuum.Mode(StorePackages), pkgs) + pkgs := generateTestPackages(3, param.RootDir) + for _, pkg := range pkgs { + err = controller.StorePackage(logE, pkg.configPkg, pkg.pkgPath) + require.NoError(t, err) + } + + // Call Close to ensure all async operations are completed + err = controller.Close(logE) require.NoError(t, err) // Run vacuum - err = controller.Vacuum(ctx, logE, vacuum.Mode(VacuumExpiredPackages), nil) + err = controller.Vacuum(logE) require.NoError(t, err) // Verify no packages were removed - err = controller.Vacuum(ctx, logE, vacuum.Mode(ListPackages), nil, "test") + err = controller.ListPackages(logE, false, "test") require.NoError(t, err) assert.Equal(t, 3, hook.LastEntry().Data["TotalPackages"]) }) } -func generateTestPackages(count int) []*config.Package { - pkgs := make([]*config.Package, count) - for i := range count { - pkgs[i] = &config.Package{ - Package: &aqua.Package{ - Name: "cli/cli", - Version: "v2." + string(rune(i+'0')) + ".0", - Registry: "standard", - }, - PackageInfo: ®istry.PackageInfo{ - Type: "github_release", - RepoOwner: "cli", - RepoName: "cli", - Asset: "gh_2." + string(rune(i+'0')) + ".0_linux_amd64.tar.gz", +func TestMockVacuumController_StorePackage(t *testing.T) { + t.Parallel() + fs := afero.NewOsFs() + testDir, err := afero.TempDir(fs, "", "vacuum_no_expired") + require.NoError(t, err) + + days := 30 + param := &config.Param{ + VacuumDays: days, + RootDir: testDir, + } + + logE := logrus.NewEntry(logrus.New()) + mockCtrl := vacuum.NewMockVacuumController() + + pkgs := generateTestPackages(2, param.RootDir) + + tests := []struct { + name string + pkg *config.Package + pkgPath string + wantErr bool + }{ + { + name: "valid package", + pkg: pkgs[0].configPkg, + pkgPath: pkgs[0].pkgPath, + wantErr: false, + }, + { + name: "nil package", + pkg: nil, + pkgPath: pkgs[1].pkgPath, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := mockCtrl.StorePackage(logE, tt.pkg, tt.pkgPath) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + err = mockCtrl.Vacuum(logE) + assert.NoError(t, err) + err = mockCtrl.Close(logE) + assert.NoError(t, err) + + }) + } +} + +func TestNilVacuumController(t *testing.T) { + t.Parallel() + logE := logrus.NewEntry(logrus.New()) + mockCtrl := &vacuum.NilVacuumController{} + + test := generateTestPackages(1, "/tmp") + err := mockCtrl.StorePackage(logE, test[0].configPkg, test[0].pkgPath) + assert.NoError(t, err) + err = mockCtrl.Vacuum(logE) + assert.NoError(t, err) + err = mockCtrl.Close(logE) + assert.NoError(t, err) + +} + +type ConfigPackageWithPath struct { + configPkg *config.Package + pkgPath string +} + +func generateTestPackages(count int, rootDir string) []ConfigPackageWithPath { + pkgs := make([]ConfigPackageWithPath, count) + for i := range pkgs { + pkgType := "github_release" + pkgName := "cli/cli" + version := "v2." + string(rune(i+'0')) + ".0" + asset := "gh_2." + string(rune(i+'0')) + ".0_linux_amd64.tar.gz" + pkgs[i] = ConfigPackageWithPath{ + configPkg: &config.Package{ + Package: &aqua.Package{ + Name: pkgName, + Version: version, + Registry: "standard", + }, + PackageInfo: ®istry.PackageInfo{ + Type: pkgType, + RepoOwner: "cli", + RepoName: "cli", + Asset: asset, + }, }, + pkgPath: filepath.Join(rootDir, "pkgs", pkgType, "github.com", pkgName, asset), } } return pkgs @@ -476,22 +487,23 @@ func BenchmarkVacuum_OnlyOneStorePackage(b *testing.B) { } // benchmarkVacuumStorePackages is a helper function to benchmark the performance of storing packages. -func benchmarkVacuumStorePackages(b *testing.B, pkgCount int) { //nolint:cyclop,funlen,gocognit +func benchmarkVacuumStorePackages(b *testing.B, pkgCount int) { b.Helper() - pkgs, logE, syncParam, syncMultipleParam, asyncParam, asyncMultipleParam, fs, syncf, syncMultiplef, asyncf, asyncMultiplef := setupBenchmark(b, pkgCount) + logE := logrus.NewEntry(logrus.New()) + fs := afero.NewOsFs() + + // Benchmark sync configuration + syncf, errf := afero.TempDir(fs, "/tmp", "vacuum_test_sync") + if errf != nil { + b.Fatal(errf) + } + pkgs := generateTestPackages(pkgCount, syncf) + vacuumDays := 5 + syncParam := &config.Param{RootDir: syncf, VacuumDays: vacuumDays} defer func() { if err := fs.RemoveAll(syncf); err != nil { b.Fatal(err) } - if err := fs.RemoveAll(syncMultiplef); err != nil { - b.Fatal(err) - } - if err := fs.RemoveAll(asyncf); err != nil { - b.Fatal(err) - } - if err := fs.RemoveAll(asyncMultiplef); err != nil { - b.Fatal(err) - } }() b.Run("Sync", func(b *testing.B) { @@ -499,87 +511,11 @@ func benchmarkVacuumStorePackages(b *testing.B, pkgCount int) { //nolint:cyclop, b.ResetTimer() for range b.N { for _, pkg := range pkgs { - if err := controller.Vacuum(context.Background(), logE, vacuum.Mode(StorePackage), []*config.Package{pkg}); err != nil { + if err := controller.StorePackage(logE, pkg.configPkg, pkg.pkgPath); err != nil { b.Fatal(err) } } + controller.Close(logE) // Close to ensure async operations are completed } }) - - b.Run("SyncMultipleSameTime", func(b *testing.B) { - controller := vacuum.New(syncMultipleParam, fs) - b.ResetTimer() - for range b.N { - if err := controller.Vacuum(context.Background(), logE, vacuum.Mode(StorePackage), pkgs); err != nil { - b.Fatal(err) - } - } - }) - - b.Run("Async", func(b *testing.B) { - controller := vacuum.New(asyncParam, fs) - b.ResetTimer() - for range b.N { - runtime.MemProfileRate = 1 - for _, pkg := range pkgs { - if err := controller.Vacuum(context.Background(), logE, vacuum.Mode(AsyncStorePackage), []*config.Package{pkg}); err != nil { - b.Fatal(err) - } - } - if err := controller.Vacuum(context.Background(), logE, vacuum.Mode(Close), nil); err != nil { - b.Fatal(err) - } - } - }) - - b.Run("AsyncMultiple", func(b *testing.B) { - controller := vacuum.New(asyncMultipleParam, fs) - b.ResetTimer() - for range b.N { - if err := controller.Vacuum(context.Background(), logE, vacuum.Mode(AsyncStorePackage), pkgs); err != nil { - b.Fatal(err) - } - if err := controller.Vacuum(context.Background(), logE, vacuum.Mode(Close), nil); err != nil { - b.Fatal(err) - } - } - }) -} - -func setupBenchmark(b *testing.B, pkgCount int) ([]*config.Package, *logrus.Entry, *config.Param, *config.Param, *config.Param, *config.Param, afero.Fs, string, string, string, string) { - b.Helper() - pkgs := generateTestPackages(pkgCount) - vacuumDays := 5 - logE := logrus.NewEntry(logrus.New()) - fs := afero.NewOsFs() - - // Benchmark sync configuration - syncf, errf := afero.TempDir(fs, "/tmp", "vacuum_test_sync") - if errf != nil { - b.Fatal(errf) - } - syncParam := &config.Param{RootDir: syncf, VacuumDays: &vacuumDays} - - // Benchmark SyncMultipleSameTime configuration - syncMultiplef, errf := afero.TempDir(fs, "/tmp", "vacuum_test_multiple") - if errf != nil { - b.Fatal(errf) - } - syncMultipleParam := &config.Param{RootDir: syncMultiplef, VacuumDays: &vacuumDays} - - // Benchmark async configuration - asyncf, errf := afero.TempDir(fs, "/tmp", "vacuum_test_async") - if errf != nil { - b.Fatal(errf) - } - asyncParam := &config.Param{RootDir: asyncf, VacuumDays: &vacuumDays} - - // Benchmark async multiple configuration - asyncMultiplef, errf := afero.TempDir(fs, "/tmp", "vacuum_test_async_multiple") - if errf != nil { - b.Fatal(errf) - } - asyncMultipleParam := &config.Param{RootDir: asyncMultiplef, VacuumDays: &vacuumDays} - - return pkgs, logE, syncParam, syncMultipleParam, asyncParam, asyncMultipleParam, fs, syncf, syncMultiplef, asyncf, asyncMultiplef } From 7290f5579cde9aaab5ce3a24d7690439ae68a35a Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Fri, 10 Jan 2025 14:09:26 +0100 Subject: [PATCH 17/73] chore(vacuum): refactor vacuum closing logic and update controller interface --- pkg/controller/exec/controller.go | 11 ++- pkg/controller/exec/exec.go | 15 +--- pkg/controller/exec/exec_test.go | 111 +------------------------ pkg/controller/install/controller.go | 11 ++- pkg/controller/install/install.go | 15 +--- pkg/controller/install/install_test.go | 4 +- pkg/controller/wire.go | 25 ++++-- pkg/installpackage/aqua_test.go | 3 +- pkg/installpackage/installer.go | 48 ++++------- pkg/installpackage/installer_test.go | 5 +- pkg/installpackage/proxy_test.go | 3 +- 11 files changed, 62 insertions(+), 189 deletions(-) diff --git a/pkg/controller/exec/controller.go b/pkg/controller/exec/controller.go index d4c1fec5c..6a6813e67 100644 --- a/pkg/controller/exec/controller.go +++ b/pkg/controller/exec/controller.go @@ -7,7 +7,6 @@ import ( "runtime" "github.com/aquaproj/aqua/v2/pkg/config" - "github.com/aquaproj/aqua/v2/pkg/controller/vacuum" "github.com/aquaproj/aqua/v2/pkg/controller/which" "github.com/aquaproj/aqua/v2/pkg/installpackage" "github.com/aquaproj/aqua/v2/pkg/osexec" @@ -27,14 +26,14 @@ type Controller struct { fs afero.Fs policyReader PolicyReader enabledXSysExec bool - vacuumCtrl *vacuum.Controller + vacuum VacuumController } type Installer interface { InstallPackage(ctx context.Context, logE *logrus.Entry, param *installpackage.ParamInstallPackage) error } -func New(pkgInstaller Installer, whichCtrl WhichController, executor Executor, osEnv osenv.OSEnv, fs afero.Fs, policyReader PolicyReader, vacuumCtrl *vacuum.Controller) *Controller { +func New(pkgInstaller Installer, whichCtrl WhichController, executor Executor, osEnv osenv.OSEnv, fs afero.Fs, policyReader PolicyReader, vacuumCtrl VacuumController) *Controller { return &Controller{ stdin: os.Stdin, stdout: os.Stdout, @@ -45,7 +44,7 @@ func New(pkgInstaller Installer, whichCtrl WhichController, executor Executor, o enabledXSysExec: getEnabledXSysExec(osEnv, runtime.GOOS), fs: fs, policyReader: policyReader, - vacuumCtrl: vacuumCtrl, + vacuum: vacuumCtrl, } } @@ -62,3 +61,7 @@ type PolicyReader interface { type WhichController interface { Which(ctx context.Context, logE *logrus.Entry, param *config.Param, exeName string) (*which.FindResult, error) } + +type VacuumController interface { + Close(logE *logrus.Entry) error +} diff --git a/pkg/controller/exec/exec.go b/pkg/controller/exec/exec.go index f0a78ebb0..2c50e206b 100644 --- a/pkg/controller/exec/exec.go +++ b/pkg/controller/exec/exec.go @@ -8,7 +8,6 @@ import ( "github.com/aquaproj/aqua/v2/pkg/checksum" "github.com/aquaproj/aqua/v2/pkg/config" - "github.com/aquaproj/aqua/v2/pkg/controller/vacuum" "github.com/aquaproj/aqua/v2/pkg/controller/which" "github.com/aquaproj/aqua/v2/pkg/installpackage" "github.com/aquaproj/aqua/v2/pkg/osexec" @@ -63,19 +62,13 @@ func (c *Controller) Exec(ctx context.Context, logE *logrus.Entry, param *config if err := c.install(ctx, logE, findResult, policyCfgs, param); err != nil { return err } - c.vacuumClose(ctx, logE) // Ensure that the vacuum process and db are closed + c.vacuumClose(logE) return c.execCommandWithRetry(ctx, logE, findResult.ExePath, args...) } -func (c *Controller) vacuumClose(ctx context.Context, logE *logrus.Entry) { - if c.vacuumCtrl == nil { - return - } - if err := c.vacuumCtrl.Vacuum(ctx, logE, vacuum.Close, nil); err != nil { - // If the closing vacuum db failed, we should not stop the process - // so we log the error and continue the process. - // Updating vacuum db will be retried next time. - logE.WithError(err).Error("close the vacuum db failed") +func (c *Controller) vacuumClose(logE *logrus.Entry) { + if err := c.vacuum.Close(logE); err != nil { + logerr.WithError(logE, err).Error("close the vacuum") } } diff --git a/pkg/controller/exec/exec_test.go b/pkg/controller/exec/exec_test.go index 636e06025..45aa0c476 100644 --- a/pkg/controller/exec/exec_test.go +++ b/pkg/controller/exec/exec_test.go @@ -127,56 +127,6 @@ packages: "../foo/gh": "/usr/local/bin/gh", }, }, - { - name: "vacuumEnabled", - rt: &runtime.Runtime{ - GOOS: "linux", - GOARCH: "amd64", - }, - param: &config.Param{ - PWD: "/home/foo/workspace", - ConfigFilePath: "aqua.yaml", - RootDir: "/home/foo/.local/share/aquaproj-aqua", - MaxParallelism: 5, - VacuumDays: func(i int) *int { return &i }(1), - }, - exeName: "aqua-installer", - dirs: []string{ - "/home/foo/workspace/.git", - }, - files: map[string]string{ - "/home/foo/workspace/aqua.yaml": `registries: -- type: local - name: standard - path: registry.yaml -packages: -- name: aquaproj/aqua-installer@v1.0.0 -`, - "/home/foo/workspace/registry.yaml": `packages: -- type: github_content - repo_owner: aquaproj - repo_name: aqua-installer - path: aqua-installer -`, - "/home/foo/.local/share/aquaproj-aqua/pkgs/github_content/github.com/aquaproj/aqua-installer/v1.0.0/aqua-installer/aqua-installer": "", - "/home/foo/workspace/aqua-policy.yaml": ` -registries: -- type: local - name: standard - path: registry.yaml -packages: -- type: local -`, - "/home/foo/.local/share/aquaproj-aqua/policies/home/foo/workspace/aqua-policy.yaml": ` -registries: -- type: local - name: standard - path: registry.yaml -packages: -- type: local -`, - }, - }, } logE := logrus.NewEntry(logrus.New()) ctx := context.Background() @@ -198,9 +148,9 @@ packages: whichCtrl := which.New(d.param, finder.NewConfigFinder(fs), reader.New(fs, d.param), registry.New(d.param, ghDownloader, fs, d.rt, &cosign.MockVerifier{}, &slsa.MockVerifier{}), d.rt, osEnv, fs, linker) downloader := download.NewDownloader(nil, download.NewHTTPDownloader(http.DefaultClient)) executor := &osexec.Mock{} - pkgInstaller := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}) + vacuumCtrl := &vacuum.MockVacuumController{} + pkgInstaller := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}, vacuumCtrl) policyFinder := policy.NewConfigFinder(fs) - vacuumCtrl := vacuum.New(d.param, fs) ctrl := execCtrl.New(pkgInstaller, whichCtrl, executor, osEnv, fs, policy.NewReader(fs, policy.NewValidator(d.param, fs), policyFinder, policy.NewConfigReader(fs)), vacuumCtrl) if err := ctrl.Exec(ctx, logE, d.param, d.exeName, d.args...); err != nil { if d.isErr { @@ -274,59 +224,6 @@ registries: path: registry.yaml packages: - type: local -`, - }, - }, - { - name: "vacuumEnabled", - rt: &runtime.Runtime{ - GOOS: "linux", - GOARCH: "amd64", - }, - param: &config.Param{ - PWD: "/home/foo/workspace", - ConfigFilePath: "aqua.yaml", - RootDir: "/home/foo/.local/share/aquaproj-aqua", - MaxParallelism: 5, - VacuumDays: func(i int) *int { return &i }(1), - }, - exeName: "aqua-installer", - env: map[string]string{ - "AQUA_VACUUM_DAYS": "1", - }, - dirs: []string{ - "/home/foo/workspace/.git", - }, - files: map[string]string{ - "/home/foo/workspace/aqua.yaml": `registries: -- type: local - name: standard - path: registry.yaml -packages: -- name: aquaproj/aqua-installer@v1.0.0 -`, - "/home/foo/workspace/registry.yaml": `packages: -- type: github_content - repo_owner: aquaproj - repo_name: aqua-installer - path: aqua-installer -`, - "/home/foo/.local/share/aquaproj-aqua/pkgs/github_content/github.com/aquaproj/aqua-installer/v1.0.0/aqua-installer/aqua-installer": "", - "/home/foo/workspace/aqua-policy.yaml": ` -registries: -- type: local - name: standard - path: registry.yaml -packages: -- type: local -`, - "/home/foo/.local/share/aquaproj-aqua/policies/home/foo/workspace/aqua-policy.yaml": ` -registries: -- type: local - name: standard - path: registry.yaml -packages: -- type: local `, }, }, @@ -351,9 +248,9 @@ packages: whichCtrl := which.New(d.param, finder.NewConfigFinder(fs), reader.New(fs, d.param), registry.New(d.param, ghDownloader, fs, d.rt, &cosign.MockVerifier{}, &slsa.MockVerifier{}), d.rt, osEnv, fs, linker) downloader := download.NewDownloader(nil, download.NewHTTPDownloader(http.DefaultClient)) executor := &osexec.Mock{} - pkgInstaller := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}) + vacuumCtrl := &vacuum.MockVacuumController{} + pkgInstaller := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}, vacuumCtrl) policyFinder := policy.NewConfigFinder(fs) - vacuumCtrl := vacuum.New(d.param, fs) ctrl := execCtrl.New(pkgInstaller, whichCtrl, executor, osEnv, fs, policy.NewReader(fs, policy.NewValidator(d.param, fs), policyFinder, policy.NewConfigReader(fs)), vacuumCtrl) b.ResetTimer() for range b.N { diff --git a/pkg/controller/install/controller.go b/pkg/controller/install/controller.go index 05d2c1523..f945ff0a6 100644 --- a/pkg/controller/install/controller.go +++ b/pkg/controller/install/controller.go @@ -7,7 +7,6 @@ import ( "github.com/aquaproj/aqua/v2/pkg/config" "github.com/aquaproj/aqua/v2/pkg/config/aqua" "github.com/aquaproj/aqua/v2/pkg/config/registry" - "github.com/aquaproj/aqua/v2/pkg/controller/vacuum" "github.com/aquaproj/aqua/v2/pkg/installpackage" "github.com/aquaproj/aqua/v2/pkg/policy" "github.com/aquaproj/aqua/v2/pkg/runtime" @@ -27,10 +26,10 @@ type Controller struct { excludedTags map[string]struct{} policyReader PolicyReader skipLink bool - vacuumCtrl *vacuum.Controller + vacuum VacuumController } -func New(param *config.Param, configFinder ConfigFinder, configReader ConfigReader, registInstaller RegistryInstaller, pkgInstaller Installer, fs afero.Fs, rt *runtime.Runtime, policyReader PolicyReader, vacuumCtrl *vacuum.Controller) *Controller { +func New(param *config.Param, configFinder ConfigFinder, configReader ConfigReader, registInstaller RegistryInstaller, pkgInstaller Installer, fs afero.Fs, rt *runtime.Runtime, policyReader PolicyReader, vacuumCtrl VacuumController) *Controller { return &Controller{ rootDir: param.RootDir, configFinder: configFinder, @@ -43,7 +42,7 @@ func New(param *config.Param, configFinder ConfigFinder, configReader ConfigRead tags: param.Tags, excludedTags: param.ExcludedTags, policyReader: policyReader, - vacuumCtrl: vacuumCtrl, + vacuum: vacuumCtrl, } } @@ -65,3 +64,7 @@ type PolicyReader interface { type RegistryInstaller interface { InstallRegistries(ctx context.Context, logE *logrus.Entry, cfg *aqua.Config, cfgFilePath string, checksums *checksum.Checksums) (map[string]*registry.Config, error) } + +type VacuumController interface { + Close(logE *logrus.Entry) error +} diff --git a/pkg/controller/install/install.go b/pkg/controller/install/install.go index 026b8f71e..adc68e807 100644 --- a/pkg/controller/install/install.go +++ b/pkg/controller/install/install.go @@ -9,7 +9,6 @@ import ( "github.com/aquaproj/aqua/v2/pkg/config" finder "github.com/aquaproj/aqua/v2/pkg/config-finder" "github.com/aquaproj/aqua/v2/pkg/config/aqua" - "github.com/aquaproj/aqua/v2/pkg/controller/vacuum" "github.com/aquaproj/aqua/v2/pkg/installpackage" "github.com/aquaproj/aqua/v2/pkg/osfile" "github.com/aquaproj/aqua/v2/pkg/policy" @@ -86,7 +85,7 @@ func (c *Controller) installAll(ctx context.Context, logE *logrus.Entry, param * } func (c *Controller) install(ctx context.Context, logE *logrus.Entry, cfgFilePath string, policyConfigs []*policy.Config, param *config.Param) error { - defer c.vacuumClose(ctx, logE) + defer c.vacuum.Close(logE) cfg := &aqua.Config{} if cfgFilePath == "" { return finder.ErrConfigFileNotFound @@ -130,15 +129,3 @@ func (c *Controller) install(ctx context.Context, logE *logrus.Entry, cfgFilePat DisablePolicy: param.DisablePolicy, }) } - -func (c *Controller) vacuumClose(ctx context.Context, logE *logrus.Entry) { - if c.vacuumCtrl == nil { - return - } - if err := c.vacuumCtrl.Vacuum(ctx, logE, vacuum.Close, nil); err != nil { - // If the closing vacuum db failed, we should not stop the process - // so we log the error and continue the process. - // Updating vacuum db will be retried next time. - logE.WithError(err).Error("close the vacuum db failed") - } -} diff --git a/pkg/controller/install/install_test.go b/pkg/controller/install/install_test.go index f0ffe040b..25e81ef67 100644 --- a/pkg/controller/install/install_test.go +++ b/pkg/controller/install/install_test.go @@ -104,10 +104,10 @@ packages: } downloader := download.NewDownloader(nil, download.NewHTTPDownloader(http.DefaultClient)) executor := &osexec.Mock{} - pkgInstaller := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}) + vacuum := vacuum.New(d.param, fs) + pkgInstaller := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}, vacuum) policyFinder := policy.NewConfigFinder(fs) policyReader := policy.NewReader(fs, &policy.MockValidator{}, policyFinder, policy.NewConfigReader(fs)) - vacuum := vacuum.New(d.param, fs) ctrl := install.New(d.param, finder.NewConfigFinder(fs), reader.New(fs, d.param), registry.New(d.param, registryDownloader, fs, d.rt, &cosign.MockVerifier{}, &slsa.MockVerifier{}), pkgInstaller, fs, d.rt, policyReader, vacuum) if err := ctrl.Install(ctx, logE, d.param); err != nil { if d.isErr { diff --git a/pkg/controller/wire.go b/pkg/controller/wire.go index be501b403..390531939 100644 --- a/pkg/controller/wire.go +++ b/pkg/controller/wire.go @@ -255,7 +255,6 @@ func InitializeInstallCommandController(ctx context.Context, param *config.Param ), wire.NewSet( installpackage.New, - installpackage.ProvideControllerOptions, wire.Bind(new(install.Installer), new(*installpackage.Installer)), ), wire.NewSet( @@ -350,7 +349,11 @@ func InitializeInstallCommandController(ctx context.Context, param *config.Param installpackage.NewCargoPackageInstallerImpl, wire.Bind(new(installpackage.CargoPackageInstaller), new(*installpackage.CargoPackageInstallerImpl)), ), - vacuum.New, + wire.NewSet( + vacuum.New, + wire.Bind(new(install.VacuumController), new(*vacuum.Controller)), + wire.Bind(new(installpackage.VacuumController), new(*vacuum.Controller)), + ), ) return &install.Controller{}, nil } @@ -427,7 +430,6 @@ func InitializeExecCommandController(ctx context.Context, param *config.Param, h ), wire.NewSet( installpackage.New, - installpackage.ProvideControllerOptions, wire.Bind(new(cexec.Installer), new(*installpackage.Installer)), ), wire.NewSet( @@ -542,7 +544,11 @@ func InitializeExecCommandController(ctx context.Context, param *config.Param, h installpackage.NewCargoPackageInstallerImpl, wire.Bind(new(installpackage.CargoPackageInstaller), new(*installpackage.CargoPackageInstallerImpl)), ), - vacuum.New, + wire.NewSet( + vacuum.New, + wire.Bind(new(cexec.VacuumController), new(*vacuum.Controller)), + wire.Bind(new(installpackage.VacuumController), new(*vacuum.Controller)), + ), ) return &cexec.Controller{}, nil } @@ -561,7 +567,6 @@ func InitializeUpdateAquaCommandController(ctx context.Context, param *config.Pa ), wire.NewSet( installpackage.New, - installpackage.ProvideControllerOptions, wire.Bind(new(updateaqua.AquaInstaller), new(*installpackage.Installer)), ), download.NewHTTPDownloader, @@ -637,6 +642,7 @@ func InitializeUpdateAquaCommandController(ctx context.Context, param *config.Pa wire.Bind(new(installpackage.CargoPackageInstaller), new(*installpackage.CargoPackageInstallerImpl)), ), provideNilVacuumController, // vacuum controller is not used so we provide nil but it is required by installPackage + wire.Bind(new(installpackage.VacuumController), new(*vacuum.NilVacuumController)), ) return &updateaqua.Controller{}, nil } @@ -659,7 +665,6 @@ func InitializeCopyCommandController(ctx context.Context, param *config.Param, h ), wire.NewSet( installpackage.New, - installpackage.ProvideControllerOptions, wire.Bind(new(install.Installer), new(*installpackage.Installer)), wire.Bind(new(cp.PackageInstaller), new(*installpackage.Installer)), ), @@ -778,7 +783,9 @@ func InitializeCopyCommandController(ctx context.Context, param *config.Param, h installpackage.NewCargoPackageInstallerImpl, wire.Bind(new(installpackage.CargoPackageInstaller), new(*installpackage.CargoPackageInstallerImpl)), ), - provideNilVacuumController, // vacuum controller is not used so we provide nil but it is required by installer and installpackage + provideNilVacuumController, // vacuum controller is not used so we provide nil but it is required by installPackage + wire.Bind(new(install.VacuumController), new(*vacuum.NilVacuumController)), + wire.Bind(new(installpackage.VacuumController), new(*vacuum.NilVacuumController)), ) return &cp.Controller{}, nil } @@ -1058,6 +1065,6 @@ func InitializeVacuumCommandController(ctx context.Context, param *config.Param, return &vacuum.Controller{} } -func provideNilVacuumController() *vacuum.Controller { - return nil +func provideNilVacuumController() *vacuum.NilVacuumController { + return &vacuum.NilVacuumController{} } diff --git a/pkg/installpackage/aqua_test.go b/pkg/installpackage/aqua_test.go index bd211f9f4..845c0be1b 100644 --- a/pkg/installpackage/aqua_test.go +++ b/pkg/installpackage/aqua_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/aquaproj/aqua/v2/pkg/config" + "github.com/aquaproj/aqua/v2/pkg/controller/vacuum" "github.com/aquaproj/aqua/v2/pkg/cosign" "github.com/aquaproj/aqua/v2/pkg/download" "github.com/aquaproj/aqua/v2/pkg/ghattestation" @@ -68,7 +69,7 @@ e922723678f493216c2398f3f23fb027c9a98808b49f6fce401ef82ee2c22b03 aqua_linux_arm } ctrl := installpackage.New(d.param, &download.Mock{ RC: io.NopCloser(strings.NewReader("xxx")), - }, d.rt, fs, installpackage.NewMockLinker(fs), d.checksumDownloader, d.checksumCalculator, &unarchive.MockUnarchiver{}, &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}) + }, d.rt, fs, installpackage.NewMockLinker(fs), d.checksumDownloader, d.checksumCalculator, &unarchive.MockUnarchiver{}, &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}, &vacuum.MockVacuumController{}) if err := ctrl.InstallAqua(ctx, logE, d.version); err != nil { if d.isErr { return diff --git a/pkg/installpackage/installer.go b/pkg/installpackage/installer.go index 18e435d1d..928656d94 100644 --- a/pkg/installpackage/installer.go +++ b/pkg/installpackage/installer.go @@ -10,7 +10,6 @@ import ( "github.com/aquaproj/aqua/v2/pkg/config" "github.com/aquaproj/aqua/v2/pkg/config/aqua" "github.com/aquaproj/aqua/v2/pkg/config/registry" - "github.com/aquaproj/aqua/v2/pkg/controller/vacuum" "github.com/aquaproj/aqua/v2/pkg/cosign" "github.com/aquaproj/aqua/v2/pkg/download" "github.com/aquaproj/aqua/v2/pkg/ghattestation" @@ -58,12 +57,12 @@ type Installer struct { cosignDisabled bool slsaDisabled bool gaaDisabled bool - VacuumCtrl *vacuum.Controller + vacuum VacuumController } -func New(param *config.Param, downloader download.ClientAPI, rt *runtime.Runtime, fs afero.Fs, linker Linker, chkDL download.ChecksumDownloader, chkCalc ChecksumCalculator, unarchiver Unarchiver, cosignVerifier CosignVerifier, slsaVerifier SLSAVerifier, minisignVerifier MinisignVerifier, ghVerifier GitHubArtifactAttestationsVerifier, goInstallInstaller GoInstallInstaller, goBuildInstaller GoBuildInstaller, cargoPackageInstaller CargoPackageInstaller, opts ...Option) *Installer { +func New(param *config.Param, downloader download.ClientAPI, rt *runtime.Runtime, fs afero.Fs, linker Linker, chkDL download.ChecksumDownloader, chkCalc ChecksumCalculator, unarchiver Unarchiver, cosignVerifier CosignVerifier, slsaVerifier SLSAVerifier, minisignVerifier MinisignVerifier, ghVerifier GitHubArtifactAttestationsVerifier, goInstallInstaller GoInstallInstaller, goBuildInstaller GoBuildInstaller, cargoPackageInstaller CargoPackageInstaller, vacuumCtrl VacuumController) *Installer { ni := func(rt *runtime.Runtime) *Installer { - return newInstaller(param, downloader, rt, fs, linker, chkDL, chkCalc, unarchiver, cosignVerifier, slsaVerifier, minisignVerifier, ghVerifier, goInstallInstaller, goBuildInstaller, cargoPackageInstaller) + return newInstaller(param, downloader, rt, fs, linker, chkDL, chkCalc, unarchiver, cosignVerifier, slsaVerifier, minisignVerifier, ghVerifier, goInstallInstaller, goBuildInstaller, cargoPackageInstaller, vacuumCtrl) } installer := ni(rt) installer.cosignInstaller = newDedicatedInstaller( @@ -86,14 +85,10 @@ func New(param *config.Param, downloader download.ClientAPI, rt *runtime.Runtime ghattestation.Package, ghattestation.Checksums(), ) - // Apply options - for _, opt := range opts { - opt(installer) - } return installer } -func newInstaller(param *config.Param, downloader download.ClientAPI, rt *runtime.Runtime, fs afero.Fs, linker Linker, chkDL download.ChecksumDownloader, chkCalc ChecksumCalculator, unarchiver Unarchiver, cosignVerifier CosignVerifier, slsaVerifier SLSAVerifier, minisignVerifier MinisignVerifier, ghVerifier GitHubArtifactAttestationsVerifier, goInstallInstaller GoInstallInstaller, goBuildInstaller GoBuildInstaller, cargoPackageInstaller CargoPackageInstaller) *Installer { +func newInstaller(param *config.Param, downloader download.ClientAPI, rt *runtime.Runtime, fs afero.Fs, linker Linker, chkDL download.ChecksumDownloader, chkCalc ChecksumCalculator, unarchiver Unarchiver, cosignVerifier CosignVerifier, slsaVerifier SLSAVerifier, minisignVerifier MinisignVerifier, ghVerifier GitHubArtifactAttestationsVerifier, goInstallInstaller GoInstallInstaller, goBuildInstaller GoBuildInstaller, cargoPackageInstaller CargoPackageInstaller, vacuumCtrl VacuumController) *Installer { return &Installer{ rootDir: param.RootDir, maxParallelism: param.MaxParallelism, @@ -118,27 +113,13 @@ func newInstaller(param *config.Param, downloader download.ClientAPI, rt *runtim goInstallInstaller: goInstallInstaller, goBuildInstaller: goBuildInstaller, cargoPackageInstaller: cargoPackageInstaller, + vacuum: vacuumCtrl, } } -// ProvideControllerOptions creates the controller options -func ProvideControllerOptions(vacuumCtrl *vacuum.Controller) []Option { - if vacuumCtrl == nil { - return []Option{} - } - return []Option{WithVacuumController(vacuumCtrl)} -} - -// Option defines a function type for controller configuration -type Option func(*Installer) - -// WithVacuumController sets the VacuumController for the Installer -func WithVacuumController(vacuumCtrl *vacuum.Controller) Option { - return func(c *Installer) { - c.VacuumCtrl = vacuumCtrl - } +type VacuumController interface { + StorePackage(logE *logrus.Entry, pkg *config.Package, pkgPath string) error } - type Linker interface { Lstat(s string) (os.FileInfo, error) Symlink(dest, src string) error @@ -302,6 +283,10 @@ func (is *Installer) InstallPackage(ctx context.Context, logE *logrus.Entry, par return fmt.Errorf("get the package install path: %w", err) } + if err := is.vacuum.StorePackage(logE, pkg, pkgPath); err != nil { + logerr.WithError(logE, err).Error("store the package") + } + if err := is.downloadWithRetry(ctx, logE, &DownloadParam{ Package: pkg, Dest: pkgPath, @@ -310,16 +295,11 @@ func (is *Installer) InstallPackage(ctx context.Context, logE *logrus.Entry, par RequireChecksum: param.RequireChecksum, Checksum: param.Checksum, }); err != nil { + // if download failed, we don't remove key from vacuum database, + // because the entry will be vacuumed by the next vacuuming, + // when this package will be considered expired return err } - // Optionally stores the package information in vacuum DB if vacuum controler is instantied and vacuum Enabled - if is.VacuumCtrl != nil && is.VacuumCtrl.IsVacuumEnabled(logE) { - logE.Debug("store package in vacuum") - if err := is.VacuumCtrl.Vacuum(ctx, logE, vacuum.AsyncStorePackage, []*config.Package{pkg}); err != nil { - logE.WithError(err).Error("store package in vacuum during install") - } - } - return is.checkFilesWrap(ctx, logE, param, pkgPath) } diff --git a/pkg/installpackage/installer_test.go b/pkg/installpackage/installer_test.go index 0c049ecff..5f905caea 100644 --- a/pkg/installpackage/installer_test.go +++ b/pkg/installpackage/installer_test.go @@ -9,6 +9,7 @@ import ( "github.com/aquaproj/aqua/v2/pkg/config" "github.com/aquaproj/aqua/v2/pkg/config/aqua" "github.com/aquaproj/aqua/v2/pkg/config/registry" + "github.com/aquaproj/aqua/v2/pkg/controller/vacuum" "github.com/aquaproj/aqua/v2/pkg/cosign" "github.com/aquaproj/aqua/v2/pkg/download" "github.com/aquaproj/aqua/v2/pkg/ghattestation" @@ -188,7 +189,7 @@ func Test_installer_InstallPackages(t *testing.T) { //nolint:funlen } } downloader := download.NewDownloader(nil, download.NewHTTPDownloader(http.DefaultClient)) - ctrl := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(d.executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}) + ctrl := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(d.executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}, &vacuum.MockVacuumController{}) if err := ctrl.InstallPackages(ctx, logE, &installpackage.ParamInstallPackages{ Config: d.cfg, Registries: d.registries, @@ -263,7 +264,7 @@ func Test_installer_InstallPackage(t *testing.T) { //nolint:funlen t.Fatal(err) } downloader := download.NewDownloader(nil, download.NewHTTPDownloader(http.DefaultClient)) - ctrl := installpackage.New(d.param, downloader, d.rt, fs, nil, nil, &checksum.Calculator{}, unarchive.New(d.executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}) + ctrl := installpackage.New(d.param, downloader, d.rt, fs, nil, nil, &checksum.Calculator{}, unarchive.New(d.executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}, &vacuum.MockVacuumController{}) if err := ctrl.InstallPackage(ctx, logE, &installpackage.ParamInstallPackage{ Pkg: d.pkg, }); err != nil { diff --git a/pkg/installpackage/proxy_test.go b/pkg/installpackage/proxy_test.go index 8b571d1f9..e4906aa72 100644 --- a/pkg/installpackage/proxy_test.go +++ b/pkg/installpackage/proxy_test.go @@ -8,6 +8,7 @@ import ( "github.com/aquaproj/aqua/v2/pkg/checksum" "github.com/aquaproj/aqua/v2/pkg/config" + "github.com/aquaproj/aqua/v2/pkg/controller/vacuum" "github.com/aquaproj/aqua/v2/pkg/cosign" "github.com/aquaproj/aqua/v2/pkg/download" "github.com/aquaproj/aqua/v2/pkg/ghattestation" @@ -64,7 +65,7 @@ func Test_installer_InstallProxy(t *testing.T) { } } downloader := download.NewDownloader(nil, download.NewHTTPDownloader(http.DefaultClient)) - ctrl := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(d.executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}) + ctrl := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(d.executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}, &vacuum.MockVacuumController{}) if err := ctrl.InstallProxy(ctx, logE); err != nil { if d.isErr { return From b5a7651e209fcce8bcf28f89bef58d0136f64fc2 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Fri, 10 Jan 2025 14:09:49 +0100 Subject: [PATCH 18/73] chore(controller): update wire_gen.go via cmdx wire --- pkg/controller/wire_gen.go | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/pkg/controller/wire_gen.go b/pkg/controller/wire_gen.go index bcdb08198..112c81e99 100644 --- a/pkg/controller/wire_gen.go +++ b/pkg/controller/wire_gen.go @@ -151,13 +151,12 @@ func InitializeInstallCommandController(ctx context.Context, param *config.Param goBuildInstallerImpl := installpackage.NewGoBuildInstallerImpl(executor) cargoPackageInstallerImpl := installpackage.NewCargoPackageInstallerImpl(executor, fs) controller := vacuum.New(param, fs) - v := installpackage.ProvideControllerOptions(controller) - installpackageInstaller := installpackage.New(param, downloader, rt, fs, linker, checksumDownloaderImpl, calculator, unarchiver, verifier, slsaVerifier, minisignVerifier, ghattestationVerifier, goInstallInstallerImpl, goBuildInstallerImpl, cargoPackageInstallerImpl, v...) + installpackageInstaller := installpackage.New(param, downloader, rt, fs, linker, checksumDownloaderImpl, calculator, unarchiver, verifier, slsaVerifier, minisignVerifier, ghattestationVerifier, goInstallInstallerImpl, goBuildInstallerImpl, cargoPackageInstallerImpl, controller) validatorImpl := policy.NewValidator(param, fs) configFinderImpl := policy.NewConfigFinder(fs) configReaderImpl := policy.NewConfigReader(fs) policyReader := policy.NewReader(fs, validatorImpl, configFinderImpl, configReaderImpl) - installController := install.New(param, configFinder, configReader, installer, installpackageInstaller, fs, rt, policyReader) + installController := install.New(param, configFinder, configReader, installer, installpackageInstaller, fs, rt, policyReader, controller) return installController, nil } @@ -207,8 +206,7 @@ func InitializeExecCommandController(ctx context.Context, param *config.Param, h goBuildInstallerImpl := installpackage.NewGoBuildInstallerImpl(executor) cargoPackageInstallerImpl := installpackage.NewCargoPackageInstallerImpl(executor, fs) controller := vacuum.New(param, fs) - v := installpackage.ProvideControllerOptions(controller) - installer := installpackage.New(param, downloader, rt, fs, linker, checksumDownloaderImpl, calculator, unarchiver, verifier, slsaVerifier, minisignVerifier, ghattestationVerifier, goInstallInstallerImpl, goBuildInstallerImpl, cargoPackageInstallerImpl, v...) + installer := installpackage.New(param, downloader, rt, fs, linker, checksumDownloaderImpl, calculator, unarchiver, verifier, slsaVerifier, minisignVerifier, ghattestationVerifier, goInstallInstallerImpl, goBuildInstallerImpl, cargoPackageInstallerImpl, controller) configFinder := finder.NewConfigFinder(fs) configReader := reader.New(fs, param) gitHubContentFileDownloader := download.NewGitHubContentFileDownloader(repositoriesService, httpDownloader) @@ -249,11 +247,10 @@ func InitializeUpdateAquaCommandController(ctx context.Context, param *config.Pa goInstallInstallerImpl := installpackage.NewGoInstallInstallerImpl(executor) goBuildInstallerImpl := installpackage.NewGoBuildInstallerImpl(executor) cargoPackageInstallerImpl := installpackage.NewCargoPackageInstallerImpl(executor, fs) - controller := provideNilVacuumController() - v := installpackage.ProvideControllerOptions(controller) - installer := installpackage.New(param, downloader, rt, fs, linker, checksumDownloaderImpl, calculator, unarchiver, verifier, slsaVerifier, minisignVerifier, ghattestationVerifier, goInstallInstallerImpl, goBuildInstallerImpl, cargoPackageInstallerImpl, v...) - updateaquaController := updateaqua.New(param, fs, rt, repositoriesService, installer) - return updateaquaController, nil + nilVacuumController := provideNilVacuumController() + installer := installpackage.New(param, downloader, rt, fs, linker, checksumDownloaderImpl, calculator, unarchiver, verifier, slsaVerifier, minisignVerifier, ghattestationVerifier, goInstallInstallerImpl, goBuildInstallerImpl, cargoPackageInstallerImpl, nilVacuumController) + controller := updateaqua.New(param, fs, rt, repositoriesService, installer) + return controller, nil } func InitializeCopyCommandController(ctx context.Context, param *config.Param, httpClient *http.Client, rt *runtime.Runtime) (*cp.Controller, error) { @@ -282,21 +279,20 @@ func InitializeCopyCommandController(ctx context.Context, param *config.Param, h goInstallInstallerImpl := installpackage.NewGoInstallInstallerImpl(executor) goBuildInstallerImpl := installpackage.NewGoBuildInstallerImpl(executor) cargoPackageInstallerImpl := installpackage.NewCargoPackageInstallerImpl(executor, fs) - controller := provideNilVacuumController() - v := installpackage.ProvideControllerOptions(controller) - installer := installpackage.New(param, downloader, rt, fs, linker, checksumDownloaderImpl, calculator, unarchiver, verifier, slsaVerifier, minisignVerifier, ghattestationVerifier, goInstallInstallerImpl, goBuildInstallerImpl, cargoPackageInstallerImpl, v...) + nilVacuumController := provideNilVacuumController() + installer := installpackage.New(param, downloader, rt, fs, linker, checksumDownloaderImpl, calculator, unarchiver, verifier, slsaVerifier, minisignVerifier, ghattestationVerifier, goInstallInstallerImpl, goBuildInstallerImpl, cargoPackageInstallerImpl, nilVacuumController) configFinder := finder.NewConfigFinder(fs) configReader := reader.New(fs, param) gitHubContentFileDownloader := download.NewGitHubContentFileDownloader(repositoriesService, httpDownloader) registryInstaller := registry.New(param, gitHubContentFileDownloader, fs, rt, verifier, slsaVerifier) osEnv := osenv.New() - whichController := which.New(param, configFinder, configReader, registryInstaller, rt, osEnv, fs, linker) + controller := which.New(param, configFinder, configReader, registryInstaller, rt, osEnv, fs, linker) validatorImpl := policy.NewValidator(param, fs) configFinderImpl := policy.NewConfigFinder(fs) configReaderImpl := policy.NewConfigReader(fs) policyReader := policy.NewReader(fs, validatorImpl, configFinderImpl, configReaderImpl) - installController := install.New(param, configFinder, configReader, registryInstaller, installer, fs, rt, policyReader) - cpController := cp.New(param, installer, fs, rt, whichController, installController, policyReader) + installController := install.New(param, configFinder, configReader, registryInstaller, installer, fs, rt, policyReader, nilVacuumController) + cpController := cp.New(param, installer, fs, rt, controller, installController, policyReader) return cpController, nil } @@ -399,6 +395,6 @@ func InitializeVacuumCommandController(ctx context.Context, param *config.Param, // wire.go: -func provideNilVacuumController() *vacuum.Controller { - return nil +func provideNilVacuumController() *vacuum.NilVacuumController { + return &vacuum.NilVacuumController{} } From 2cb4eb0a78692e940d43fef0288bfd48b3ea9f76 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Fri, 10 Jan 2025 14:33:32 +0100 Subject: [PATCH 19/73] chore(mod): update go.mod with go mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index b07b4c5c1..3032c5d17 100644 --- a/go.mod +++ b/go.mod @@ -78,7 +78,7 @@ require ( github.com/shopspring/decimal v1.4.0 // indirect github.com/sorairolake/lzip-go v0.3.5 // indirect github.com/spf13/cast v1.7.0 // indirect - github.com/stretchr/testify v1.9.0 // indirect + github.com/stretchr/testify v1.9.0 github.com/suzuki-shunsuke/go-jsoneq v0.1.2 // indirect github.com/therootcompany/xz v1.0.1 // indirect github.com/ulikunitz/xz v0.5.12 // indirect From 84636e48a51979ecb0942561ede4e5b33fa734da Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Fri, 10 Jan 2025 14:36:11 +0100 Subject: [PATCH 20/73] tests(vacuum): fix lint tests --- pkg/controller/vacuum/vacuum_test.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pkg/controller/vacuum/vacuum_test.go b/pkg/controller/vacuum/vacuum_test.go index 439f78376..23102854a 100644 --- a/pkg/controller/vacuum/vacuum_test.go +++ b/pkg/controller/vacuum/vacuum_test.go @@ -414,17 +414,17 @@ func TestMockVacuumController_StorePackage(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Parallel() err := mockCtrl.StorePackage(logE, tt.pkg, tt.pkgPath) if tt.wantErr { - assert.Error(t, err) + require.Error(t, err) } else { - assert.NoError(t, err) + require.NoError(t, err) } err = mockCtrl.Vacuum(logE) - assert.NoError(t, err) + require.NoError(t, err) err = mockCtrl.Close(logE) - assert.NoError(t, err) - + require.NoError(t, err) }) } } @@ -436,12 +436,11 @@ func TestNilVacuumController(t *testing.T) { test := generateTestPackages(1, "/tmp") err := mockCtrl.StorePackage(logE, test[0].configPkg, test[0].pkgPath) - assert.NoError(t, err) + require.NoError(t, err) err = mockCtrl.Vacuum(logE) - assert.NoError(t, err) + require.NoError(t, err) err = mockCtrl.Close(logE) - assert.NoError(t, err) - + require.NoError(t, err) } type ConfigPackageWithPath struct { From 24c573f4ce8465fbb30eb521b4b47eca3454b986 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Fri, 10 Jan 2025 14:38:13 +0100 Subject: [PATCH 21/73] tests(vacuum): remove unused param --- pkg/controller/vacuum/vacuum_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/controller/vacuum/vacuum_test.go b/pkg/controller/vacuum/vacuum_test.go index 23102854a..fa61430d3 100644 --- a/pkg/controller/vacuum/vacuum_test.go +++ b/pkg/controller/vacuum/vacuum_test.go @@ -381,10 +381,8 @@ func TestMockVacuumController_StorePackage(t *testing.T) { testDir, err := afero.TempDir(fs, "", "vacuum_no_expired") require.NoError(t, err) - days := 30 param := &config.Param{ - VacuumDays: days, - RootDir: testDir, + RootDir: testDir, } logE := logrus.NewEntry(logrus.New()) From a8b1d21cddc37cfd91e4c2e0359618d94c072852 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Fri, 10 Jan 2025 14:43:27 +0100 Subject: [PATCH 22/73] tests(vacuum): remove unused const --- pkg/controller/vacuum/vacuum_test.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/pkg/controller/vacuum/vacuum_test.go b/pkg/controller/vacuum/vacuum_test.go index fa61430d3..5627caf8b 100644 --- a/pkg/controller/vacuum/vacuum_test.go +++ b/pkg/controller/vacuum/vacuum_test.go @@ -16,16 +16,6 @@ import ( "github.com/stretchr/testify/require" ) -const ( - ListPackages string = "list-packages" - ListExpiredPackages string = "list-expired-packages" - StorePackage string = "store-package" - StorePackages string = "store-packages" - AsyncStorePackage string = "async-store-package" - VacuumExpiredPackages string = "vacuum-expired-packages" - Close string = "close" -) - func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop t.Parallel() // Setup common test fixtures From 11ff3359c98b44e8a41283f2f45e75136733aa2d Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 07:43:01 +0100 Subject: [PATCH 23/73] chore(cli): remove default vacuum days setting --- pkg/cli/util/util.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/cli/util/util.go b/pkg/cli/util/util.go index 265294da9..87ba5efe3 100644 --- a/pkg/cli/util/util.go +++ b/pkg/cli/util/util.go @@ -125,7 +125,6 @@ func setEnvParams(param *config.Param) error { //nolint:cyclop } } } - param.VacuumDays = 0 // Disabled by default if a := os.Getenv("AQUA_VACUUM_DAYS"); a != "" { vacuumDays, err := strconv.Atoi(a) if err != nil { From 470a28eef770e61cde42197390b7c844016e26a8 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 07:47:02 +0100 Subject: [PATCH 24/73] docs(vacuum): update command descriptions for clarity --- pkg/cli/vacuum/command.go | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/pkg/cli/vacuum/command.go b/pkg/cli/vacuum/command.go index ac1f7d9cc..80fa08d4b 100644 --- a/pkg/cli/vacuum/command.go +++ b/pkg/cli/vacuum/command.go @@ -13,24 +13,16 @@ import ( ) const description = `Perform vacuuming tasks. -If no argument is provided, the vacuum will clean expired packages. - - # Execute vacuum cleaning - $ aqua vacuum - -This command has an alias "v". - - $ aqua v Enable vacuuming by setting the AQUA_VACUUM_DAYS environment variable to a value greater than 0. This command removes versions of packages that have not been used for the specified number of days. You can list all packages managed by the vacuum system or only expired packages. - # List all packages managed by the vacuum system + # Show all packages managed by the vacuum system $ aqua vacuum show - # List only expired packages + # Show only expired packages $ aqua vacuum show --expired $ aqua vacuum show -e @@ -95,7 +87,7 @@ func (i *command) action(c *cli.Context) error { if c.Command.Name == "show" { if err := ctrl.ListPackages(i.r.LogE, c.Bool("expired")); err != nil { - return fmt.Errorf("list packages: %w", err) + return fmt.Errorf("show packages: %w", err) } return nil } From 6a1887936d3d58fe2025ad5d31f580f3fdecb32d Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 07:48:48 +0100 Subject: [PATCH 25/73] fix(controller): use filepath.Join for constructing database file path --- pkg/controller/vacuum/controller.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/controller/vacuum/controller.go b/pkg/controller/vacuum/controller.go index 06ad0454a..1bcf82852 100644 --- a/pkg/controller/vacuum/controller.go +++ b/pkg/controller/vacuum/controller.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "os" + "path/filepath" "sync" "sync/atomic" "time" @@ -49,7 +50,7 @@ func (vc *Controller) getDB() (*bolt.DB, error) { } const dbFileMode = 0o600 - db, err := bolt.Open(vc.Param.RootDir+"/"+dbFile, dbFileMode, &bolt.Options{ + db, err := bolt.Open(filepath.Join(vc.Param.RootDir, dbFile), dbFileMode, &bolt.Options{ Timeout: 1 * time.Second, }) if err != nil { From f394fe9bc44b5344611067114a3b43fef075324d Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 07:52:07 +0100 Subject: [PATCH 26/73] fix(controller): improve error logging during database operation retries --- pkg/controller/vacuum/controller.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/controller/vacuum/controller.go b/pkg/controller/vacuum/controller.go index 1bcf82852..14f5ed256 100644 --- a/pkg/controller/vacuum/controller.go +++ b/pkg/controller/vacuum/controller.go @@ -86,10 +86,7 @@ func (vc *Controller) withDBRetry(logE *logrus.Entry, fn func(*bolt.Tx) error, d return nil } - logE.WithFields(logrus.Fields{ - "attempt": i + 1, - "error": err, - }).Warn("retrying database operation") + logerr.WithError(logE, err).WithField("attempt", i+1).Warn("retrying database operation") time.Sleep(backoff) backoff *= exponentialBackoff From d9e093e58f631da2dc31722c47a7e6f71ca8b1d8 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 07:53:21 +0100 Subject: [PATCH 27/73] fix(controller): simplify error handling in database update and view operations --- pkg/controller/vacuum/controller.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/controller/vacuum/controller.go b/pkg/controller/vacuum/controller.go index 14f5ed256..8969e70a7 100644 --- a/pkg/controller/vacuum/controller.go +++ b/pkg/controller/vacuum/controller.go @@ -111,14 +111,12 @@ func (vc *Controller) withDB(logE *logrus.Entry, fn func(*bolt.Tx) error, dbAcce }() if dbAccessType == Update { - err = db.Update(fn) - if err != nil { + if err := db.Update(fn); err != nil { return fmt.Errorf("update database: %w", err) } return nil } - err = db.View(fn) - if err != nil { + if err := db.View(fn); err != nil { return fmt.Errorf("view database: %w", err) } return nil From db190744ee4aa32101e3fd668d5bdc903b6f8b5a Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 07:54:25 +0100 Subject: [PATCH 28/73] fix(controller): use filepath.Join for database file path construction in TestKeepDBOpen --- pkg/controller/vacuum/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/vacuum/controller.go b/pkg/controller/vacuum/controller.go index 8969e70a7..71cb90a3b 100644 --- a/pkg/controller/vacuum/controller.go +++ b/pkg/controller/vacuum/controller.go @@ -125,7 +125,7 @@ func (vc *Controller) withDB(logE *logrus.Entry, fn func(*bolt.Tx) error, dbAcce // Keep_DBOpen opens the database instance. This is used for testing purposes. func (vc *Controller) TestKeepDBOpen() error { const dbFileMode = 0o600 - _, _ = bolt.Open(vc.Param.RootDir+"/"+dbFile, dbFileMode, &bolt.Options{ + _, _ = bolt.Open(filepath.Join(vc.Param.RootDir, dbFile), dbFileMode, &bolt.Options{ Timeout: 1 * time.Second, }) return nil From 4325fbdd1bebfb3f877562441c4038afd7993d14 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 07:57:32 +0100 Subject: [PATCH 29/73] fix(controller): enhance error handling in TestKeepDBOpen and update test to use require.NoError --- pkg/controller/vacuum/controller.go | 6 ++++-- pkg/controller/vacuum/vacuum_test.go | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/controller/vacuum/controller.go b/pkg/controller/vacuum/controller.go index 71cb90a3b..4431486ee 100644 --- a/pkg/controller/vacuum/controller.go +++ b/pkg/controller/vacuum/controller.go @@ -125,9 +125,11 @@ func (vc *Controller) withDB(logE *logrus.Entry, fn func(*bolt.Tx) error, dbAcce // Keep_DBOpen opens the database instance. This is used for testing purposes. func (vc *Controller) TestKeepDBOpen() error { const dbFileMode = 0o600 - _, _ = bolt.Open(filepath.Join(vc.Param.RootDir, dbFile), dbFileMode, &bolt.Options{ + if _, err := bolt.Open(filepath.Join(vc.Param.RootDir, dbFile), dbFileMode, &bolt.Options{ Timeout: 1 * time.Second, - }) + }); err != nil { + return fmt.Errorf("open database %v: %w", dbFile, err) + } return nil } diff --git a/pkg/controller/vacuum/vacuum_test.go b/pkg/controller/vacuum/vacuum_test.go index 5627caf8b..67922bada 100644 --- a/pkg/controller/vacuum/vacuum_test.go +++ b/pkg/controller/vacuum/vacuum_test.go @@ -102,9 +102,9 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop pkgs := generateTestPackages(numberPackagesToStore, param.RootDir) // We force Keeping the DB open to simulate a failure in the async operation - if err := controller.TestKeepDBOpen(); err != nil { - t.Fatal(err) - } + err = controller.TestKeepDBOpen() + require.NoError(t, err) + hook.Reset() for _, pkg := range pkgs { err := controller.StorePackage(logE, pkg.configPkg, pkg.pkgPath) From a67cb3a71cd4945bd26a319bb9e0b1605cc8f433 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 08:00:25 +0100 Subject: [PATCH 30/73] fix(controller): improve error logging in package decoding and simplify error message in package removal --- pkg/controller/vacuum/vacuum.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/controller/vacuum/vacuum.go b/pkg/controller/vacuum/vacuum.go index 685e50443..28a93b3cf 100644 --- a/pkg/controller/vacuum/vacuum.go +++ b/pkg/controller/vacuum/vacuum.go @@ -186,9 +186,7 @@ func (vc *Controller) listPackages(logE *logrus.Entry) ([]*PackageVacuumEntry, e return b.ForEach(func(k, value []byte) error { pkgEntry, err := decodePackageEntry(value) if err != nil { - logE.WithFields(logrus.Fields{ - "pkg_key": string(k), - }).Warnf("unable to decode entry: %v", err) + logerr.WithError(logE, err).WithField("pkg_key", string(k)).Warn("unable to decode entry") return err } pkgs = append(pkgs, &PackageVacuumEntry{ @@ -302,7 +300,7 @@ func (vc *Controller) vacuumExpiredPackages(logE *logrus.Entry) error { defer vc.Close(logE) if len(successfulRemovals) > 0 { if err := vc.removePackages(logE, successfulRemovals); err != nil { - return fmt.Errorf("failed to remove packages from database: %w", err) + return fmt.Errorf("remove packages from database: %w", err) } } From 8b0b7cd001f469a2ba8a14fad90277ba5047a1ad Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 08:17:58 +0100 Subject: [PATCH 31/73] fix(controller): enhance error logging for package removal and improve error message for encountered issues --- pkg/controller/vacuum/vacuum.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/pkg/controller/vacuum/vacuum.go b/pkg/controller/vacuum/vacuum.go index 28a93b3cf..7f65042fe 100644 --- a/pkg/controller/vacuum/vacuum.go +++ b/pkg/controller/vacuum/vacuum.go @@ -294,7 +294,12 @@ func (vc *Controller) vacuumExpiredPackages(logE *logrus.Entry) error { successfulRemovals, errorsEncountered := vc.processExpiredPackages(logE, expiredPackages) if len(errorsEncountered) > 0 { - return errors.New("some packages could not be removed") + var gErr error + for _, err := range errorsEncountered { + logerr.WithError(logE, err).Error("removing package path from system") + } + gErr = fmt.Errorf("total of %d errors encountered while removing package paths", len(errorsEncountered)) + return gErr } defer vc.Close(logE) @@ -309,14 +314,6 @@ func (vc *Controller) vacuumExpiredPackages(logE *logrus.Entry) error { // processExpiredPackages processes a list of expired package entries by removing their associated paths // and generating a list of configuration packages to be removed from vacuum database. -// -// Parameters: -// - logE: A logrus.Entry used for logging errors and information. -// - expired: A slice of PackageVacuumEntry representing the expired packages to be processed. -// -// Returns: -// - A slice of VacuumPackage representing the packages that were successfully processed and need to be removed from the vacuum database. -// - A slice of errors encountered during the processing of the expired packages. func (vc *Controller) processExpiredPackages(logE *logrus.Entry, expired []*PackageVacuumEntry) ([]string, []error) { const batchSize = 10 successKeys := make(chan string, len(expired)) @@ -354,7 +351,7 @@ func (vc *Controller) processExpiredPackages(logE *logrus.Entry, expired []*Pack defer wg.Done() for _, entry := range batch { if err := vc.removePackageVersionPath(vc.Param, entry.pkgPath, entry.pkgType); err != nil { - logerr.WithError(logE, err).Error("removing path") + logerr.WithError(logE, err).WithField("pkg_path", entry.pkgPath).Error("removing path") errCh <- err continue } From 20011277b7e5983df4a38e4c889e02870a58acd5 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 08:22:32 +0100 Subject: [PATCH 32/73] fix(controller): update logging fields for package storage to improve clarity and consistency --- pkg/controller/vacuum/vacuum.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/controller/vacuum/vacuum.go b/pkg/controller/vacuum/vacuum.go index 7f65042fe..e8d4fffb1 100644 --- a/pkg/controller/vacuum/vacuum.go +++ b/pkg/controller/vacuum/vacuum.go @@ -389,9 +389,9 @@ func (vc *Controller) storePackageInternal(logE *logrus.Entry, pkg *Package, dat return errors.New("bucket not found") } logE.WithFields(logrus.Fields{ - "name": pkg.Name, - "version": pkg.Version, - "pkg_path": pkg.PkgPath, + "package_name": pkg.Name, + "package_version": pkg.Version, + "package_path": pkg.PkgPath, }).Debug("storing package in vacuum database") pkgKey := pkg.PkgPath @@ -404,15 +404,15 @@ func (vc *Controller) storePackageInternal(logE *logrus.Entry, pkg *Package, dat if err != nil { logerr.WithError(logE, err).WithFields( logrus.Fields{ - "name": pkg.Name, - "version": pkg.Version, - "pkg_path": pkg.PkgPath, + "package_name": pkg.Name, + "package_version": pkg.Version, + "package_path": pkg.PkgPath, }).Error("encode package") return fmt.Errorf("encode package %s: %w", pkg.Name, err) } if err := b.Put([]byte(pkgKey), data); err != nil { - logerr.WithError(logE, err).WithField("pkgKey", pkgKey).Error("store package in vacuum database") + logerr.WithError(logE, err).WithField("package_path", pkgKey).Error("store package in vacuum database") return fmt.Errorf("store package %s: %w", pkg.Name, err) } return nil From 81df2bd056e32991a46b0a4ef5e6987cb74a1cc2 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 08:22:54 +0100 Subject: [PATCH 33/73] fix(controller): simplify error messages in package display and encoding functions for clarity --- pkg/controller/vacuum/vacuum.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/controller/vacuum/vacuum.go b/pkg/controller/vacuum/vacuum.go index e8d4fffb1..9ab236ee6 100644 --- a/pkg/controller/vacuum/vacuum.go +++ b/pkg/controller/vacuum/vacuum.go @@ -273,7 +273,7 @@ func (vc *Controller) displayPackagesFuzzyInteractive(pkgs []*PackageVacuumEntry if errors.Is(err, fuzzyfinder.ErrAbort) { return nil } - return fmt.Errorf("failed to display packages: %w", err) + return fmt.Errorf("display packages: %w", err) } return nil @@ -467,7 +467,7 @@ func (vc *Controller) removePackageVersionPath(param *config.Param, path string, func encodePackageEntry(pkgEntry *PackageEntry) ([]byte, error) { data, err := json.Marshal(pkgEntry) if err != nil { - return nil, fmt.Errorf("failed to marshal package entry: %w", err) + return nil, fmt.Errorf("marshal package entry: %w", err) } return data, nil } @@ -476,7 +476,7 @@ func encodePackageEntry(pkgEntry *PackageEntry) ([]byte, error) { func decodePackageEntry(data []byte) (*PackageEntry, error) { var pkgEntry PackageEntry if err := json.Unmarshal(data, &pkgEntry); err != nil { - return nil, fmt.Errorf("failed to unmarshal package entry: %w", err) + return nil, fmt.Errorf("unmarshal package entry: %w", err) } return &pkgEntry, nil } From d8ae573f28afa569f31137032ded30600f62ecc6 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 08:44:16 +0100 Subject: [PATCH 34/73] fix(controller): remove parent directory cleanup to keep aqua simple --- pkg/controller/vacuum/vacuum.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/pkg/controller/vacuum/vacuum.go b/pkg/controller/vacuum/vacuum.go index 9ab236ee6..a49acbe47 100644 --- a/pkg/controller/vacuum/vacuum.go +++ b/pkg/controller/vacuum/vacuum.go @@ -13,7 +13,6 @@ import ( "github.com/dustin/go-humanize" "github.com/ktr0731/go-fuzzyfinder" "github.com/sirupsen/logrus" - "github.com/spf13/afero" "github.com/suzuki-shunsuke/logrus-error/logerr" "go.etcd.io/bbolt" ) @@ -440,26 +439,10 @@ func (vc *Controller) removePackages(logE *logrus.Entry, pkgs []string) error { // removePackageVersionPath removes the specified package version directory and its parent directory if it becomes empty. func (vc *Controller) removePackageVersionPath(param *config.Param, path string, pkgType string) error { pkgsPath := filepath.Join(param.RootDir, "pkgs") - pkgsRemoveLimitPath := filepath.Join(pkgsPath, pkgType) pkgVersionPath := filepath.Join(pkgsPath, path) if err := vc.fs.RemoveAll(pkgVersionPath); err != nil { return fmt.Errorf("remove package version directories: %w", err) } - - currentPath := filepath.Dir(pkgVersionPath) - for currentPath != pkgsRemoveLimitPath { - dirIsEmpty, err := afero.IsEmpty(vc.fs, currentPath) - if err != nil { - return fmt.Errorf("check if directory is empty: %w", err) - } - if !dirIsEmpty { - break - } - if err := vc.fs.RemoveAll(currentPath); err != nil { - return fmt.Errorf("remove package directories: %w", err) - } - currentPath = filepath.Dir(currentPath) - } return nil } From 1b71a5707275bd0fb0712117bedf1836fec18534 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 09:06:06 +0100 Subject: [PATCH 35/73] chore(lint): fix lint error --- pkg/controller/vacuum/vacuum.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pkg/controller/vacuum/vacuum.go b/pkg/controller/vacuum/vacuum.go index a49acbe47..7ca166b3a 100644 --- a/pkg/controller/vacuum/vacuum.go +++ b/pkg/controller/vacuum/vacuum.go @@ -327,14 +327,12 @@ func (vc *Controller) processExpiredPackages(logE *logrus.Entry, expired []*Pack batch := make([]struct { pkgPath string - pkgType string pkgName string version string }, len(expired[i:end])) for j, entry := range expired[i:end] { batch[j].pkgPath = string(entry.PkgPath) - batch[j].pkgType = entry.PackageEntry.Package.Type batch[j].pkgName = entry.PackageEntry.Package.Name batch[j].version = entry.PackageEntry.Package.Version } @@ -342,14 +340,13 @@ func (vc *Controller) processExpiredPackages(logE *logrus.Entry, expired []*Pack wg.Add(1) go func(batch []struct { pkgPath string - pkgType string pkgName string version string }, ) { defer wg.Done() for _, entry := range batch { - if err := vc.removePackageVersionPath(vc.Param, entry.pkgPath, entry.pkgType); err != nil { + if err := vc.removePackageVersionPath(vc.Param, entry.pkgPath); err != nil { logerr.WithError(logE, err).WithField("pkg_path", entry.pkgPath).Error("removing path") errCh <- err continue @@ -437,7 +434,7 @@ func (vc *Controller) removePackages(logE *logrus.Entry, pkgs []string) error { } // removePackageVersionPath removes the specified package version directory and its parent directory if it becomes empty. -func (vc *Controller) removePackageVersionPath(param *config.Param, path string, pkgType string) error { +func (vc *Controller) removePackageVersionPath(param *config.Param, path string) error { pkgsPath := filepath.Join(param.RootDir, "pkgs") pkgVersionPath := filepath.Join(pkgsPath, path) if err := vc.fs.RemoveAll(pkgVersionPath); err != nil { From d2000a8263a1e8a253a4766b1cc08b0df76d33c8 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 09:21:56 +0100 Subject: [PATCH 36/73] test(controller): add unit tests for package entry encoding and decoding --- pkg/controller/vacuum/vacuum_internal_test.go | 61 +++++++++++++++++++ pkg/controller/vacuum/vacuum_test.go | 6 ++ 2 files changed, 67 insertions(+) create mode 100644 pkg/controller/vacuum/vacuum_internal_test.go diff --git a/pkg/controller/vacuum/vacuum_internal_test.go b/pkg/controller/vacuum/vacuum_internal_test.go new file mode 100644 index 000000000..a7daa2451 --- /dev/null +++ b/pkg/controller/vacuum/vacuum_internal_test.go @@ -0,0 +1,61 @@ +package vacuum + +import ( + "encoding/json" + "testing" + "time" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +func TestHandleAsyncStorePackage_NilPackage(t *testing.T) { + logE := logrus.NewEntry(logrus.New()) + + vacuumCtrl := New(nil, nil) + + // Test + err := vacuumCtrl.handleAsyncStorePackage(logE, nil) + + // Assert + assert.Error(t, err) + assert.Equal(t, "vacuumPkg is nil", err.Error()) +} +func TestEncodePackageEntry(t *testing.T) { + pkgEntry := &PackageEntry{ + LastUsageTime: time.Now(), + Package: &Package{Name: "test-package"}, + } + + data, err := encodePackageEntry(pkgEntry) + assert.NoError(t, err) + assert.NotNil(t, data) + + var decodedEntry PackageEntry + err = json.Unmarshal(data, &decodedEntry) + assert.NoError(t, err) + assert.Equal(t, pkgEntry.LastUsageTime.Unix(), decodedEntry.LastUsageTime.Unix()) + assert.Equal(t, pkgEntry.Package.Name, decodedEntry.Package.Name) +} + +func TestDecodePackageEntry(t *testing.T) { + pkgEntry := &PackageEntry{ + LastUsageTime: time.Now(), + Package: &Package{Name: "test-package"}, + } + + data, err := json.Marshal(pkgEntry) + assert.NoError(t, err) + + decodedEntry, err := decodePackageEntry(data) + assert.NoError(t, err) + assert.NotNil(t, decodedEntry) + assert.Equal(t, pkgEntry.LastUsageTime.Unix(), decodedEntry.LastUsageTime.Unix()) + assert.Equal(t, pkgEntry.Package.Name, decodedEntry.Package.Name) +} + +func TestDecodePackageEntry_Error(t *testing.T) { + _, err := decodePackageEntry([]byte("invalid json")) + assert.Error(t, err) + assert.Contains(t, err.Error(), "unmarshal package entry") +} diff --git a/pkg/controller/vacuum/vacuum_test.go b/pkg/controller/vacuum/vacuum_test.go index 67922bada..ba26eca07 100644 --- a/pkg/controller/vacuum/vacuum_test.go +++ b/pkg/controller/vacuum/vacuum_test.go @@ -46,6 +46,12 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop err = controller.ListPackages(logE, false, "test") require.NoError(t, err, "Should return nil when vacuum is disabled") + + err = controller.Vacuum(logE) + require.NoError(t, err, "Should return nil when vacuum is disabled") + + err = controller.Close(logE) + require.NoError(t, err, "Should return nil when vacuum is disabled") }) t.Run("vacuum bad configuration", func(t *testing.T) { From e1d47fbbe96f587e579591bab749225ea48a6223 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 09:22:13 +0100 Subject: [PATCH 37/73] test(workflow): add integration test for vacuum run command --- .github/workflows/wc-integration-test.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/wc-integration-test.yaml b/.github/workflows/wc-integration-test.yaml index d1129f19b..c6c985b6d 100644 --- a/.github/workflows/wc-integration-test.yaml +++ b/.github/workflows/wc-integration-test.yaml @@ -217,6 +217,11 @@ jobs: - name: Test update-aqua run: aqua update-aqua + - name: Test vacuum Run command + env: + AQUA_VACUUM_DAYS: 1 + run: aqua vacuum run + integration-test-cargo: timeout-minutes: 30 runs-on: ubuntu-latest From fc4ead0361ecab51a18e02c9b43d70e9d8131f23 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 09:27:42 +0100 Subject: [PATCH 38/73] chore(vacuum): fix lint tests --- pkg/controller/vacuum/vacuum_internal_test.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pkg/controller/vacuum/vacuum_internal_test.go b/pkg/controller/vacuum/vacuum_internal_test.go index a7daa2451..1b6d9f8cb 100644 --- a/pkg/controller/vacuum/vacuum_internal_test.go +++ b/pkg/controller/vacuum/vacuum_internal_test.go @@ -7,9 +7,11 @@ import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestHandleAsyncStorePackage_NilPackage(t *testing.T) { + t.Parallel() logE := logrus.NewEntry(logrus.New()) vacuumCtrl := New(nil, nil) @@ -18,44 +20,48 @@ func TestHandleAsyncStorePackage_NilPackage(t *testing.T) { err := vacuumCtrl.handleAsyncStorePackage(logE, nil) // Assert - assert.Error(t, err) + require.Error(t, err) assert.Equal(t, "vacuumPkg is nil", err.Error()) } + func TestEncodePackageEntry(t *testing.T) { + t.Parallel() pkgEntry := &PackageEntry{ LastUsageTime: time.Now(), Package: &Package{Name: "test-package"}, } data, err := encodePackageEntry(pkgEntry) - assert.NoError(t, err) + require.NoError(t, err) assert.NotNil(t, data) var decodedEntry PackageEntry err = json.Unmarshal(data, &decodedEntry) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, pkgEntry.LastUsageTime.Unix(), decodedEntry.LastUsageTime.Unix()) assert.Equal(t, pkgEntry.Package.Name, decodedEntry.Package.Name) } func TestDecodePackageEntry(t *testing.T) { + t.Parallel() pkgEntry := &PackageEntry{ LastUsageTime: time.Now(), Package: &Package{Name: "test-package"}, } data, err := json.Marshal(pkgEntry) - assert.NoError(t, err) + require.NoError(t, err) decodedEntry, err := decodePackageEntry(data) - assert.NoError(t, err) + require.NoError(t, err) assert.NotNil(t, decodedEntry) assert.Equal(t, pkgEntry.LastUsageTime.Unix(), decodedEntry.LastUsageTime.Unix()) assert.Equal(t, pkgEntry.Package.Name, decodedEntry.Package.Name) } func TestDecodePackageEntry_Error(t *testing.T) { + t.Parallel() _, err := decodePackageEntry([]byte("invalid json")) - assert.Error(t, err) + require.Error(t, err) assert.Contains(t, err.Error(), "unmarshal package entry") } From abd0bf400a42f9a313324c0024fb6dec92b6ddcc Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 09:38:32 +0100 Subject: [PATCH 39/73] ci(vacuum): run vacuum command before update aqua --- .github/workflows/wc-integration-test.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wc-integration-test.yaml b/.github/workflows/wc-integration-test.yaml index c6c985b6d..398150fc1 100644 --- a/.github/workflows/wc-integration-test.yaml +++ b/.github/workflows/wc-integration-test.yaml @@ -214,14 +214,15 @@ jobs: - run: aqua list working-directory: /tmp - - name: Test update-aqua - run: aqua update-aqua - - name: Test vacuum Run command env: AQUA_VACUUM_DAYS: 1 run: aqua vacuum run + - name: Test update-aqua + run: aqua update-aqua + + integration-test-cargo: timeout-minutes: 30 runs-on: ubuntu-latest From 07fcc264bf67ecfbaa2827e646aead87042140c8 Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Sun, 12 Jan 2025 00:03:59 +0900 Subject: [PATCH 40/73] fix: replace time.Sleep with timer.Wait to support cancel --- pkg/cli/vacuum/command.go | 4 +- pkg/controller/install/install_test.go | 2 +- pkg/controller/vacuum/controller.go | 7 ++- pkg/controller/vacuum/queue_store.go | 11 ++-- pkg/controller/vacuum/vacuum.go | 53 ++++++++-------- pkg/controller/vacuum/vacuum_internal_test.go | 3 +- pkg/controller/vacuum/vacuum_test.go | 62 +++++++++++-------- pkg/controller/wire_gen.go | 6 +- 8 files changed, 80 insertions(+), 68 deletions(-) diff --git a/pkg/cli/vacuum/command.go b/pkg/cli/vacuum/command.go index 80fa08d4b..1ed9ea348 100644 --- a/pkg/cli/vacuum/command.go +++ b/pkg/cli/vacuum/command.go @@ -86,14 +86,14 @@ func (i *command) action(c *cli.Context) error { ctrl := controller.InitializeVacuumCommandController(c.Context, param, http.DefaultClient, i.r.Runtime) if c.Command.Name == "show" { - if err := ctrl.ListPackages(i.r.LogE, c.Bool("expired")); err != nil { + if err := ctrl.ListPackages(c.Context, i.r.LogE, c.Bool("expired")); err != nil { return fmt.Errorf("show packages: %w", err) } return nil } if c.Command.Name == "run" { - if err := ctrl.Vacuum(i.r.LogE); err != nil { + if err := ctrl.Vacuum(c.Context, i.r.LogE); err != nil { return fmt.Errorf("run: %w", err) } } diff --git a/pkg/controller/install/install_test.go b/pkg/controller/install/install_test.go index 25e81ef67..6741797c5 100644 --- a/pkg/controller/install/install_test.go +++ b/pkg/controller/install/install_test.go @@ -104,7 +104,7 @@ packages: } downloader := download.NewDownloader(nil, download.NewHTTPDownloader(http.DefaultClient)) executor := &osexec.Mock{} - vacuum := vacuum.New(d.param, fs) + vacuum := vacuum.New(ctx, d.param, fs) pkgInstaller := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}, vacuum) policyFinder := policy.NewConfigFinder(fs) policyReader := policy.NewReader(fs, &policy.MockValidator{}, policyFinder, policy.NewConfigReader(fs)) diff --git a/pkg/controller/vacuum/controller.go b/pkg/controller/vacuum/controller.go index 4431486ee..de886cb48 100644 --- a/pkg/controller/vacuum/controller.go +++ b/pkg/controller/vacuum/controller.go @@ -1,6 +1,7 @@ package vacuum import ( + "context" "fmt" "io" "os" @@ -26,13 +27,13 @@ type Controller struct { } // New initializes a Controller with the given context, parameters, and dependencies. -func New(param *config.Param, fs afero.Fs) *Controller { +func New(ctx context.Context, param *config.Param, fs afero.Fs) *Controller { vc := &Controller{ stdout: os.Stdout, Param: param, fs: fs, } - vc.storeQueue = newStoreQueue(vc) + vc.storeQueue = newStoreQueue(ctx, vc) return vc } @@ -73,7 +74,7 @@ func (vc *Controller) getDB() (*bolt.DB, error) { } // withDBRetry retries a database operation with exponential backoff. -func (vc *Controller) withDBRetry(logE *logrus.Entry, fn func(*bolt.Tx) error, dbAccessType DBAccessType) error { +func (vc *Controller) withDBRetry(ctx context.Context, logE *logrus.Entry, fn func(*bolt.Tx) error, dbAccessType DBAccessType) error { const ( retries = 2 initialBackoff = 100 * time.Millisecond diff --git a/pkg/controller/vacuum/queue_store.go b/pkg/controller/vacuum/queue_store.go index f8e69b49b..f12a842c7 100644 --- a/pkg/controller/vacuum/queue_store.go +++ b/pkg/controller/vacuum/queue_store.go @@ -1,6 +1,7 @@ package vacuum import ( + "context" "sync" "github.com/sirupsen/logrus" @@ -23,7 +24,7 @@ type StoreQueue struct { } // newStoreQueue initializes the task queue with a single worker. -func newStoreQueue(vc *Controller) *StoreQueue { +func newStoreQueue(ctx context.Context, vc *Controller) *StoreQueue { const maxTasks = 100 sq := &StoreQueue{ taskQueue: make(chan StoreRequest, maxTasks), @@ -31,19 +32,19 @@ func newStoreQueue(vc *Controller) *StoreQueue { vc: vc, } - go sq.worker() + go sq.worker(ctx) return sq } // worker processes tasks from the queue. -func (sq *StoreQueue) worker() { +func (sq *StoreQueue) worker(ctx context.Context) { for { select { case task, ok := <-sq.taskQueue: if !ok { return } - err := sq.vc.storePackageInternal(task.logE, task.pkg) + err := sq.vc.storePackageInternal(ctx, task.logE, task.pkg) if err != nil { logerr.WithError(task.logE, err).Error("store package asynchronously") } @@ -52,7 +53,7 @@ func (sq *StoreQueue) worker() { // Process remaining tasks for len(sq.taskQueue) > 0 { task := <-sq.taskQueue - err := sq.vc.storePackageInternal(task.logE, task.pkg) + err := sq.vc.storePackageInternal(ctx, task.logE, task.pkg) if err != nil { logerr.WithError(task.logE, err).Error("store package asynchronously during shutdown") } diff --git a/pkg/controller/vacuum/vacuum.go b/pkg/controller/vacuum/vacuum.go index 7ca166b3a..ff80891fb 100644 --- a/pkg/controller/vacuum/vacuum.go +++ b/pkg/controller/vacuum/vacuum.go @@ -1,6 +1,7 @@ package vacuum import ( + "context" "encoding/json" "errors" "fmt" @@ -46,26 +47,26 @@ type Package struct { } // Vacuum performs the vacuuming process if it is enabled. -func (vc *Controller) Vacuum(logE *logrus.Entry) error { +func (vc *Controller) Vacuum(ctx context.Context, logE *logrus.Entry) error { if !vc.IsVacuumEnabled(logE) { return nil } - return vc.vacuumExpiredPackages(logE) + return vc.vacuumExpiredPackages(ctx, logE) } // ListPackages lists the packages based on the provided arguments. // If the expired flag is set to true, it lists the expired packages. // Otherwise, it lists all packages. -func (vc *Controller) ListPackages(logE *logrus.Entry, expired bool, args ...string) error { +func (vc *Controller) ListPackages(ctx context.Context, logE *logrus.Entry, expired bool, args ...string) error { if expired { - return vc.handleListExpiredPackages(logE, args...) + return vc.handleListExpiredPackages(ctx, logE, args...) } - return vc.handleListPackages(logE, args...) + return vc.handleListPackages(ctx, logE, args...) } // handleListPackages retrieves a list of packages and displays them using a fuzzy search. -func (vc *Controller) handleListPackages(logE *logrus.Entry, args ...string) error { - pkgs, err := vc.listPackages(logE) +func (vc *Controller) handleListPackages(ctx context.Context, logE *logrus.Entry, args ...string) error { + pkgs, err := vc.listPackages(ctx, logE) if err != nil { return err } @@ -74,8 +75,8 @@ func (vc *Controller) handleListPackages(logE *logrus.Entry, args ...string) err // handleListExpiredPackages handles the process of listing expired packages // and displaying them using a fuzzy search. -func (vc *Controller) handleListExpiredPackages(logE *logrus.Entry, args ...string) error { - expiredPkgs, err := vc.listExpiredPackages(logE) +func (vc *Controller) handleListExpiredPackages(ctx context.Context, logE *logrus.Entry, args ...string) error { + expiredPkgs, err := vc.listExpiredPackages(ctx, logE) if err != nil { return err } @@ -135,8 +136,8 @@ func (vc *Controller) IsVacuumEnabled(logE *logrus.Entry) bool { } // listExpiredPackages lists all packages that have expired based on the vacuum configuration. -func (vc *Controller) listExpiredPackages(logE *logrus.Entry) ([]*PackageVacuumEntry, error) { - pkgs, err := vc.listPackages(logE) +func (vc *Controller) listExpiredPackages(ctx context.Context, logE *logrus.Entry) ([]*PackageVacuumEntry, error) { + pkgs, err := vc.listPackages(ctx, logE) if err != nil { return nil, err } @@ -165,7 +166,7 @@ func (vc *Controller) isPackageExpired(pkg *PackageVacuumEntry) bool { } // listPackages lists all stored package entries. -func (vc *Controller) listPackages(logE *logrus.Entry) ([]*PackageVacuumEntry, error) { +func (vc *Controller) listPackages(ctx context.Context, logE *logrus.Entry) ([]*PackageVacuumEntry, error) { db, err := vc.getDB() if err != nil { return nil, err @@ -177,7 +178,7 @@ func (vc *Controller) listPackages(logE *logrus.Entry) ([]*PackageVacuumEntry, e var pkgs []*PackageVacuumEntry - err = vc.withDBRetry(logE, func(tx *bbolt.Tx) error { + err = vc.withDBRetry(ctx, logE, func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(bucketNamePkgs)) if b == nil { return nil @@ -279,8 +280,8 @@ func (vc *Controller) displayPackagesFuzzyInteractive(pkgs []*PackageVacuumEntry } // vacuumExpiredPackages performs cleanup of expired packages. -func (vc *Controller) vacuumExpiredPackages(logE *logrus.Entry) error { - expiredPackages, err := vc.listExpiredPackages(logE) +func (vc *Controller) vacuumExpiredPackages(ctx context.Context, logE *logrus.Entry) error { + expiredPackages, err := vc.listExpiredPackages(ctx, logE) if err != nil { return err } @@ -303,7 +304,7 @@ func (vc *Controller) vacuumExpiredPackages(logE *logrus.Entry) error { defer vc.Close(logE) if len(successfulRemovals) > 0 { - if err := vc.removePackages(logE, successfulRemovals); err != nil { + if err := vc.removePackages(ctx, logE, successfulRemovals); err != nil { return fmt.Errorf("remove packages from database: %w", err) } } @@ -374,12 +375,12 @@ func (vc *Controller) processExpiredPackages(logE *logrus.Entry, expired []*Pack } // storePackageInternal stores package entries in the database. -func (vc *Controller) storePackageInternal(logE *logrus.Entry, pkg *Package, dateTime ...time.Time) error { +func (vc *Controller) storePackageInternal(ctx context.Context, logE *logrus.Entry, pkg *Package, dateTime ...time.Time) error { lastUsedTime := time.Now() if len(dateTime) > 0 { lastUsedTime = dateTime[0] } - return vc.withDBRetry(logE, func(tx *bbolt.Tx) error { + return vc.withDBRetry(ctx, logE, func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(bucketNamePkgs)) if b == nil { return errors.New("bucket not found") @@ -416,8 +417,8 @@ func (vc *Controller) storePackageInternal(logE *logrus.Entry, pkg *Package, dat } // removePackages removes package entries from the database. -func (vc *Controller) removePackages(logE *logrus.Entry, pkgs []string) error { - return vc.withDBRetry(logE, func(tx *bbolt.Tx) error { +func (vc *Controller) removePackages(ctx context.Context, logE *logrus.Entry, pkgs []string) error { + return vc.withDBRetry(ctx, logE, func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(bucketNamePkgs)) if b == nil { return errors.New("bucket not found") @@ -462,9 +463,9 @@ func decodePackageEntry(data []byte) (*PackageEntry, error) { } // GetPackageLastUsed retrieves the last used time of a package. for testing purposes. -func (vc *Controller) GetPackageLastUsed(logE *logrus.Entry, pkgPath string) *time.Time { +func (vc *Controller) GetPackageLastUsed(ctx context.Context, logE *logrus.Entry, pkgPath string) *time.Time { var lastUsedTime time.Time - pkgEntry, _ := vc.retrievePackageEntry(logE, pkgPath) + pkgEntry, _ := vc.retrievePackageEntry(ctx, logE, pkgPath) if pkgEntry != nil { lastUsedTime = pkgEntry.LastUsageTime } @@ -472,16 +473,16 @@ func (vc *Controller) GetPackageLastUsed(logE *logrus.Entry, pkgPath string) *ti } // SetTimeStampPackage permit define a Timestamp for a package Manually. for testing purposes. -func (vc *Controller) SetTimestampPackage(logE *logrus.Entry, pkg *config.Package, pkgPath string, datetime time.Time) error { +func (vc *Controller) SetTimestampPackage(ctx context.Context, logE *logrus.Entry, pkg *config.Package, pkgPath string, datetime time.Time) error { vacuumPkg := vc.getVacuumPackage(pkg, pkgPath) - return vc.storePackageInternal(logE, vacuumPkg, datetime) + return vc.storePackageInternal(ctx, logE, vacuumPkg, datetime) } // retrievePackageEntry retrieves a package entry from the database by key. for testing purposes. -func (vc *Controller) retrievePackageEntry(logE *logrus.Entry, key string) (*PackageEntry, error) { +func (vc *Controller) retrievePackageEntry(ctx context.Context, logE *logrus.Entry, key string) (*PackageEntry, error) { var pkgEntry *PackageEntry key = generatePackageKey(vc.Param.RootDir, key) - err := vc.withDBRetry(logE, func(tx *bbolt.Tx) error { + err := vc.withDBRetry(ctx, logE, func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(bucketNamePkgs)) if b == nil { return nil diff --git a/pkg/controller/vacuum/vacuum_internal_test.go b/pkg/controller/vacuum/vacuum_internal_test.go index 1b6d9f8cb..7f6c75714 100644 --- a/pkg/controller/vacuum/vacuum_internal_test.go +++ b/pkg/controller/vacuum/vacuum_internal_test.go @@ -1,6 +1,7 @@ package vacuum import ( + "context" "encoding/json" "testing" "time" @@ -14,7 +15,7 @@ func TestHandleAsyncStorePackage_NilPackage(t *testing.T) { t.Parallel() logE := logrus.NewEntry(logrus.New()) - vacuumCtrl := New(nil, nil) + vacuumCtrl := New(context.Background(), nil, nil) // Test err := vacuumCtrl.handleAsyncStorePackage(logE, nil) diff --git a/pkg/controller/vacuum/vacuum_test.go b/pkg/controller/vacuum/vacuum_test.go index ba26eca07..e35015284 100644 --- a/pkg/controller/vacuum/vacuum_test.go +++ b/pkg/controller/vacuum/vacuum_test.go @@ -1,6 +1,7 @@ package vacuum_test import ( + "context" "path/filepath" "testing" "time" @@ -42,12 +43,13 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop param := &config.Param{ RootDir: testDir, } - controller := vacuum.New(param, fs) + ctx := context.Background() + controller := vacuum.New(ctx, param, fs) - err = controller.ListPackages(logE, false, "test") + err = controller.ListPackages(ctx, logE, false, "test") require.NoError(t, err, "Should return nil when vacuum is disabled") - err = controller.Vacuum(logE) + err = controller.Vacuum(ctx, logE) require.NoError(t, err, "Should return nil when vacuum is disabled") err = controller.Close(logE) @@ -64,7 +66,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop RootDir: testDir, VacuumDays: -1, } - controller := vacuum.New(param, fs) + controller := vacuum.New(context.Background(), param, fs) err = controller.StorePackage(logE, nil, testDir) require.NoError(t, err, "Should return nil when vacuum is disabled") @@ -82,10 +84,11 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop VacuumDays: days, RootDir: testDir, } - controller := vacuum.New(param, fs) + ctx := context.Background() + controller := vacuum.New(ctx, param, fs) // Test - err = controller.ListPackages(logE, false, "test") + err = controller.ListPackages(ctx, logE, false, "test") // Assert require.NoError(t, err) // Should succeed with empty database @@ -102,7 +105,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop VacuumDays: days, RootDir: testDir, } - controller := vacuum.New(param, fs) + controller := vacuum.New(context.Background(), param, fs) numberPackagesToStore := 7 pkgs := generateTestPackages(numberPackagesToStore, param.RootDir) @@ -146,7 +149,8 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop VacuumDays: days, RootDir: testDir, } - controller := vacuum.New(param, fs) + ctx := context.Background() + controller := vacuum.New(ctx, param, fs) numberPackagesToStore := 1 pkgs := generateTestPackages(numberPackagesToStore, param.RootDir) @@ -159,7 +163,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop require.NoError(t, err) // List packages - should contain our stored package - err = controller.ListPackages(logE, false, "test") + err = controller.ListPackages(ctx, logE, false, "test") require.NoError(t, err) assert.Equal(t, "Test mode: Displaying packages", hook.LastEntry().Message) assert.Equal(t, 1, hook.LastEntry().Data["TotalPackages"]) @@ -167,7 +171,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop hook.Reset() // Verify package was stored correctly - lastUsed := controller.GetPackageLastUsed(logE, pkgs[0].pkgPath) + lastUsed := controller.GetPackageLastUsed(ctx, logE, pkgs[0].pkgPath) assert.False(t, lastUsed.IsZero(), "Package should have a last used time") }) @@ -181,7 +185,8 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop VacuumDays: days, RootDir: testDir, } - controller := vacuum.New(param, fs) + ctx := context.Background() + controller := vacuum.New(ctx, param, fs) numberPackagesToStore := 4 pkgs := generateTestPackages(numberPackagesToStore, param.RootDir) @@ -196,7 +201,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop require.NoError(t, err) // List packages - should contain our stored package - err = controller.ListPackages(logE, false, "test") + err = controller.ListPackages(ctx, logE, false, "test") require.NoError(t, err) assert.Equal(t, "Test mode: Displaying packages", hook.LastEntry().Message) assert.Equal(t, 4, hook.LastEntry().Data["TotalPackages"]) @@ -214,7 +219,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop VacuumDays: days, RootDir: testDir, } - controller := vacuum.New(param, fs) + controller := vacuum.New(context.Background(), param, fs) // Store the package err = controller.StorePackage(logE, nil, tempTestDir) @@ -232,10 +237,11 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop VacuumDays: days, RootDir: testDir, } - controller := vacuum.New(param, fs) + ctx := context.Background() + controller := vacuum.New(ctx, param, fs) // Test - err = controller.ListPackages(logE, true, "test") + err = controller.ListPackages(ctx, logE, true, "test") // Assert require.NoError(t, err) // Error if no package found @@ -255,7 +261,8 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop VacuumDays: days, RootDir: testDir, } - controller := vacuum.New(param, fs) + ctx := context.Background() + controller := vacuum.New(ctx, param, fs) numberPackagesToStore := 3 numberPackagesToExpire := 1 @@ -288,28 +295,28 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop // Modify timestamp of one package to be expired oldTime := time.Now().Add(-48 * time.Hour) // 2 days old for _, pkg := range pkgs[:numberPackagesToExpire] { - err = controller.SetTimestampPackage(logE, pkg.configPkg, pkg.pkgPath, oldTime) + err = controller.SetTimestampPackage(ctx, logE, pkg.configPkg, pkg.pkgPath, oldTime) require.NoError(t, err) } // Check Packages after expiration - err = controller.ListPackages(logE, false, "test") + err = controller.ListPackages(ctx, logE, false, "test") require.NoError(t, err) assert.Equal(t, numberPackagesToStore, hook.LastEntry().Data["TotalPackages"]) assert.Equal(t, numberPackagesToExpire, hook.LastEntry().Data["TotalExpired"]) // List expired packages only - err = controller.ListPackages(logE, true, "test") + err = controller.ListPackages(ctx, logE, true, "test") require.NoError(t, err) assert.Equal(t, numberPackagesToExpire, hook.LastEntry().Data["TotalPackages"]) assert.Equal(t, numberPackagesToExpire, hook.LastEntry().Data["TotalExpired"]) // Run vacuum - err = controller.Vacuum(logE) + err = controller.Vacuum(ctx, logE) require.NoError(t, err) // List expired packages - err = controller.ListPackages(logE, true, "test") + err = controller.ListPackages(ctx, logE, true, "test") require.NoError(t, err) assert.Equal(t, "no packages to display", hook.LastEntry().Message) @@ -322,7 +329,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop // Modify timestamp of one package to be expired And lock DB to simulate a failure in the vacuum operation for _, pkg := range pkgs[:numberPackagesToExpire] { - err = controller.SetTimestampPackage(logE, pkg.configPkg, pkg.pkgPath, oldTime) + err = controller.SetTimestampPackage(ctx, logE, pkg.configPkg, pkg.pkgPath, oldTime) require.NoError(t, err) } @@ -331,7 +338,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop require.NoError(t, err) // Run vacuum - err = controller.Vacuum(logE) + err = controller.Vacuum(ctx, logE) require.Error(t, err) assert.Contains(t, err.Error(), "open database vacuum.db: timeout") }) @@ -347,7 +354,8 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop VacuumDays: days, RootDir: testDir, } - controller := vacuum.New(param, fs) + ctx := context.Background() + controller := vacuum.New(ctx, param, fs) // Store non-expired packages pkgs := generateTestPackages(3, param.RootDir) @@ -361,11 +369,11 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop require.NoError(t, err) // Run vacuum - err = controller.Vacuum(logE) + err = controller.Vacuum(ctx, logE) require.NoError(t, err) // Verify no packages were removed - err = controller.ListPackages(logE, false, "test") + err = controller.ListPackages(ctx, logE, false, "test") require.NoError(t, err) assert.Equal(t, 3, hook.LastEntry().Data["TotalPackages"]) }) @@ -500,7 +508,7 @@ func benchmarkVacuumStorePackages(b *testing.B, pkgCount int) { }() b.Run("Sync", func(b *testing.B) { - controller := vacuum.New(syncParam, fs) + controller := vacuum.New(context.Background(), syncParam, fs) b.ResetTimer() for range b.N { for _, pkg := range pkgs { diff --git a/pkg/controller/wire_gen.go b/pkg/controller/wire_gen.go index 112c81e99..249c33249 100644 --- a/pkg/controller/wire_gen.go +++ b/pkg/controller/wire_gen.go @@ -150,7 +150,7 @@ func InitializeInstallCommandController(ctx context.Context, param *config.Param goInstallInstallerImpl := installpackage.NewGoInstallInstallerImpl(executor) goBuildInstallerImpl := installpackage.NewGoBuildInstallerImpl(executor) cargoPackageInstallerImpl := installpackage.NewCargoPackageInstallerImpl(executor, fs) - controller := vacuum.New(param, fs) + controller := vacuum.New(ctx, param, fs) installpackageInstaller := installpackage.New(param, downloader, rt, fs, linker, checksumDownloaderImpl, calculator, unarchiver, verifier, slsaVerifier, minisignVerifier, ghattestationVerifier, goInstallInstallerImpl, goBuildInstallerImpl, cargoPackageInstallerImpl, controller) validatorImpl := policy.NewValidator(param, fs) configFinderImpl := policy.NewConfigFinder(fs) @@ -205,7 +205,7 @@ func InitializeExecCommandController(ctx context.Context, param *config.Param, h goInstallInstallerImpl := installpackage.NewGoInstallInstallerImpl(executor) goBuildInstallerImpl := installpackage.NewGoBuildInstallerImpl(executor) cargoPackageInstallerImpl := installpackage.NewCargoPackageInstallerImpl(executor, fs) - controller := vacuum.New(param, fs) + controller := vacuum.New(ctx, param, fs) installer := installpackage.New(param, downloader, rt, fs, linker, checksumDownloaderImpl, calculator, unarchiver, verifier, slsaVerifier, minisignVerifier, ghattestationVerifier, goInstallInstallerImpl, goBuildInstallerImpl, cargoPackageInstallerImpl, controller) configFinder := finder.NewConfigFinder(fs) configReader := reader.New(fs, param) @@ -389,7 +389,7 @@ func InitializeRemoveCommandController(ctx context.Context, param *config.Param, func InitializeVacuumCommandController(ctx context.Context, param *config.Param, httpClient *http.Client, rt *runtime.Runtime) *vacuum.Controller { fs := afero.NewOsFs() - controller := vacuum.New(param, fs) + controller := vacuum.New(ctx, param, fs) return controller } From 94e7725ccf81847d64f08f9da7dd0a601fec42ac Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 16:08:24 +0100 Subject: [PATCH 41/73] ci(workflow): add test aqua command with vacuum enabled --- .github/workflows/wc-integration-test.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/wc-integration-test.yaml b/.github/workflows/wc-integration-test.yaml index 398150fc1..895a343c2 100644 --- a/.github/workflows/wc-integration-test.yaml +++ b/.github/workflows/wc-integration-test.yaml @@ -214,11 +214,21 @@ jobs: - run: aqua list working-directory: /tmp + # Test if commands succeed with vacuum enabled + - name: Test aqua install with AQUA_VACUUM_DAYS + env: + AQUA_VACUUM_DAYS: 1 + run: aqua install + - name: Test aqua exec with AQUA_VACUUM_DAYS + env: + AQUA_VACUUM_DAYS: 1 + run: cmdx -v - name: Test vacuum Run command env: AQUA_VACUUM_DAYS: 1 run: aqua vacuum run + # Test update-aqua command - name: Test update-aqua run: aqua update-aqua From fa7933878fa9d55be24973beef9295bfa49bb80e Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Sun, 12 Jan 2025 00:09:04 +0900 Subject: [PATCH 42/73] fix: replace time.Sleep with timer.Wait for cancel --- pkg/controller/vacuum/controller.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/controller/vacuum/controller.go b/pkg/controller/vacuum/controller.go index de886cb48..00f6782ac 100644 --- a/pkg/controller/vacuum/controller.go +++ b/pkg/controller/vacuum/controller.go @@ -11,6 +11,7 @@ import ( "time" "github.com/aquaproj/aqua/v2/pkg/config" + "github.com/aquaproj/aqua/v2/pkg/timer" "github.com/sirupsen/logrus" "github.com/spf13/afero" "github.com/suzuki-shunsuke/logrus-error/logerr" @@ -89,7 +90,7 @@ func (vc *Controller) withDBRetry(ctx context.Context, logE *logrus.Entry, fn fu logerr.WithError(logE, err).WithField("attempt", i+1).Warn("retrying database operation") - time.Sleep(backoff) + timer.Wait(ctx, backoff) backoff *= exponentialBackoff } From a66f09925582392d040feae16c3933291c8ec140 Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Sun, 12 Jan 2025 00:18:04 +0900 Subject: [PATCH 43/73] fix: handle an error of timer.Wait --- pkg/controller/vacuum/controller.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/controller/vacuum/controller.go b/pkg/controller/vacuum/controller.go index 00f6782ac..61d341206 100644 --- a/pkg/controller/vacuum/controller.go +++ b/pkg/controller/vacuum/controller.go @@ -90,7 +90,9 @@ func (vc *Controller) withDBRetry(ctx context.Context, logE *logrus.Entry, fn fu logerr.WithError(logE, err).WithField("attempt", i+1).Warn("retrying database operation") - timer.Wait(ctx, backoff) + if err := timer.Wait(ctx, backoff); err != nil { + return fmt.Errorf("wait for retrying database operation: %w", err) + } backoff *= exponentialBackoff } From fdd9ff03327cef89f8324187a7f6102b0ac0997b Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 16:49:57 +0100 Subject: [PATCH 44/73] tests(vacuum): set specific logger for each test --- pkg/controller/vacuum/vacuum_test.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/pkg/controller/vacuum/vacuum_test.go b/pkg/controller/vacuum/vacuum_test.go index e35015284..cccaf5ce2 100644 --- a/pkg/controller/vacuum/vacuum_test.go +++ b/pkg/controller/vacuum/vacuum_test.go @@ -19,9 +19,6 @@ import ( func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop t.Parallel() - // Setup common test fixtures - logger, hook := test.NewNullLogger() - logE := logrus.NewEntry(logger) fs := afero.NewOsFs() @@ -37,6 +34,8 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop t.Run("vacuum disabled", func(t *testing.T) { t.Parallel() + logger, _ := test.NewNullLogger() + logE := logrus.NewEntry(logger) testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_disabled") require.NoError(t, err) // Setup @@ -58,6 +57,8 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop t.Run("vacuum bad configuration", func(t *testing.T) { t.Parallel() + logger, hook := test.NewNullLogger() + logE := logrus.NewEntry(logger) logE.Logger.Level = logrus.DebugLevel testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_bad_config") require.NoError(t, err) @@ -75,6 +76,8 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop t.Run("ListPackages mode - empty database", func(t *testing.T) { t.Parallel() + logger, hook := test.NewNullLogger() + logE := logrus.NewEntry(logger) // Setup - use a new temp directory for this test testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_list_test") require.NoError(t, err) @@ -97,6 +100,8 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop t.Run("StoreFailed", func(t *testing.T) { t.Parallel() + logger, hook := test.NewNullLogger() + logE := logrus.NewEntry(logger) testDir, err := afero.TempDir(fs, tempTestDir, "store_failed") require.NoError(t, err) @@ -140,6 +145,8 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop t.Run("StorePackage and ListPackages workflow", func(t *testing.T) { t.Parallel() + logger, hook := test.NewNullLogger() + logE := logrus.NewEntry(logger) // Setup - use a new temp directory for this test testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_store_test") require.NoError(t, err) @@ -177,6 +184,8 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop t.Run("StoreMultiplePackages", func(t *testing.T) { t.Parallel() + logger, hook := test.NewNullLogger() + logE := logrus.NewEntry(logger) testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_StoreMultiplePackages_test") require.NoError(t, err) @@ -211,6 +220,8 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop t.Run("StoreNilPackage", func(t *testing.T) { t.Parallel() + logger, hook := test.NewNullLogger() + logE := logrus.NewEntry(logger) testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_StoreNilPackage_test") require.NoError(t, err) @@ -229,6 +240,8 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop t.Run("handleListExpiredPackages - no expired packages", func(t *testing.T) { t.Parallel() + logger, hook := test.NewNullLogger() + logE := logrus.NewEntry(logger) testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_handle_list_expired") require.NoError(t, err) @@ -249,6 +262,8 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop }) t.Run("VacuumExpiredPackages workflow", func(t *testing.T) { t.Parallel() + logger, hook := test.NewNullLogger() + logE := logrus.NewEntry(logger) testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_expire_test") require.NoError(t, err) defer func() { @@ -345,6 +360,8 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop t.Run("TestVacuumWithoutExpiredPackages", func(t *testing.T) { t.Parallel() + logger, hook := test.NewNullLogger() + logE := logrus.NewEntry(logger) fs := afero.NewOsFs() testDir, err := afero.TempDir(fs, "", "vacuum_no_expired") require.NoError(t, err) From 99f6456a154d24e00358170064b5fee7674c7bf6 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 17:17:20 +0100 Subject: [PATCH 45/73] tests(vacuum): replace assert and require with testing and go-cmp --- pkg/controller/vacuum/vacuum_internal_test.go | 63 +++- pkg/controller/vacuum/vacuum_test.go | 297 +++++++++++++----- 2 files changed, 270 insertions(+), 90 deletions(-) diff --git a/pkg/controller/vacuum/vacuum_internal_test.go b/pkg/controller/vacuum/vacuum_internal_test.go index 7f6c75714..f17a0dd6e 100644 --- a/pkg/controller/vacuum/vacuum_internal_test.go +++ b/pkg/controller/vacuum/vacuum_internal_test.go @@ -6,9 +6,8 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestHandleAsyncStorePackage_NilPackage(t *testing.T) { @@ -21,8 +20,12 @@ func TestHandleAsyncStorePackage_NilPackage(t *testing.T) { err := vacuumCtrl.handleAsyncStorePackage(logE, nil) // Assert - require.Error(t, err) - assert.Equal(t, "vacuumPkg is nil", err.Error()) + if err == nil { + t.Fatalf("expected an error but got nil") + } + if diff := cmp.Diff("vacuumPkg is nil", err.Error()); diff != "" { + t.Errorf("unexpected error message (-want +got):\n%s", diff) + } } func TestEncodePackageEntry(t *testing.T) { @@ -33,14 +36,24 @@ func TestEncodePackageEntry(t *testing.T) { } data, err := encodePackageEntry(pkgEntry) - require.NoError(t, err) - assert.NotNil(t, data) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if data == nil { + t.Fatalf("expected data but got nil") + } var decodedEntry PackageEntry err = json.Unmarshal(data, &decodedEntry) - require.NoError(t, err) - assert.Equal(t, pkgEntry.LastUsageTime.Unix(), decodedEntry.LastUsageTime.Unix()) - assert.Equal(t, pkgEntry.Package.Name, decodedEntry.Package.Name) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if diff := cmp.Diff(pkgEntry.LastUsageTime.Unix(), decodedEntry.LastUsageTime.Unix()); diff != "" { + t.Errorf("unexpected LastUsageTime (-want +got):\n%s", diff) + } + if diff := cmp.Diff(pkgEntry.Package.Name, decodedEntry.Package.Name); diff != "" { + t.Errorf("unexpected Package.Name (-want +got):\n%s", diff) + } } func TestDecodePackageEntry(t *testing.T) { @@ -51,18 +64,36 @@ func TestDecodePackageEntry(t *testing.T) { } data, err := json.Marshal(pkgEntry) - require.NoError(t, err) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } decodedEntry, err := decodePackageEntry(data) - require.NoError(t, err) - assert.NotNil(t, decodedEntry) - assert.Equal(t, pkgEntry.LastUsageTime.Unix(), decodedEntry.LastUsageTime.Unix()) - assert.Equal(t, pkgEntry.Package.Name, decodedEntry.Package.Name) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if decodedEntry == nil { + t.Fatalf("expected decodedEntry but got nil") + } + if diff := cmp.Diff(pkgEntry.LastUsageTime.Unix(), decodedEntry.LastUsageTime.Unix()); diff != "" { + t.Errorf("unexpected LastUsageTime (-want +got):\n%s", diff) + } + if diff := cmp.Diff(pkgEntry.Package.Name, decodedEntry.Package.Name); diff != "" { + t.Errorf("unexpected Package.Name (-want +got):\n%s", diff) + } } func TestDecodePackageEntry_Error(t *testing.T) { t.Parallel() _, err := decodePackageEntry([]byte("invalid json")) - require.Error(t, err) - assert.Contains(t, err.Error(), "unmarshal package entry") + if err == nil { + t.Fatalf("expected an error but got nil") + } + if diff := cmp.Diff(true, contains(err.Error(), "unmarshal package entry")); diff != "" { + t.Errorf("unexpected error message (-want +got):\n%s", diff) + } +} + +func contains(s, substr string) bool { + return len(s) >= len(substr) && s[:len(substr)] == substr } diff --git a/pkg/controller/vacuum/vacuum_test.go b/pkg/controller/vacuum/vacuum_test.go index cccaf5ce2..51fdb2b5e 100644 --- a/pkg/controller/vacuum/vacuum_test.go +++ b/pkg/controller/vacuum/vacuum_test.go @@ -10,11 +10,10 @@ import ( "github.com/aquaproj/aqua/v2/pkg/config/aqua" "github.com/aquaproj/aqua/v2/pkg/config/registry" "github.com/aquaproj/aqua/v2/pkg/controller/vacuum" + "github.com/google/go-cmp/cmp" "github.com/sirupsen/logrus" "github.com/sirupsen/logrus/hooks/test" "github.com/spf13/afero" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop @@ -24,7 +23,9 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop // Create temp directory for tests tempTestDir, err := afero.TempDir(fs, "/tmp", "vacuum_test") - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } t.Cleanup(func() { err := fs.RemoveAll(tempTestDir) if err != nil { @@ -37,7 +38,9 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop logger, _ := test.NewNullLogger() logE := logrus.NewEntry(logger) testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_disabled") - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } // Setup param := &config.Param{ RootDir: testDir, @@ -46,13 +49,19 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop controller := vacuum.New(ctx, param, fs) err = controller.ListPackages(ctx, logE, false, "test") - require.NoError(t, err, "Should return nil when vacuum is disabled") + if err != nil { + t.Fatal("Should return nil when vacuum is disabled") + } err = controller.Vacuum(ctx, logE) - require.NoError(t, err, "Should return nil when vacuum is disabled") + if err != nil { + t.Fatal("Should return nil when vacuum is disabled") + } err = controller.Close(logE) - require.NoError(t, err, "Should return nil when vacuum is disabled") + if err != nil { + t.Fatal("Should return nil when vacuum is disabled") + } }) t.Run("vacuum bad configuration", func(t *testing.T) { @@ -61,7 +70,9 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop logE := logrus.NewEntry(logger) logE.Logger.Level = logrus.DebugLevel testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_bad_config") - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } // Setup param := &config.Param{ RootDir: testDir, @@ -70,8 +81,12 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop controller := vacuum.New(context.Background(), param, fs) err = controller.StorePackage(logE, nil, testDir) - require.NoError(t, err, "Should return nil when vacuum is disabled") - assert.Equal(t, "vacuum is disabled. AQUA_VACUUM_DAYS is not set or invalid.", hook.LastEntry().Message) + if err != nil { + t.Fatal("Should return nil when vacuum is disabled") + } + if diff := cmp.Diff("vacuum is disabled. AQUA_VACUUM_DAYS is not set or invalid.", hook.LastEntry().Message); diff != "" { + t.Errorf("Unexpected log message (-want +got):\n%s", diff) + } }) t.Run("ListPackages mode - empty database", func(t *testing.T) { @@ -80,7 +95,9 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop logE := logrus.NewEntry(logger) // Setup - use a new temp directory for this test testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_list_test") - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } days := 30 param := &config.Param{ @@ -94,8 +111,12 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop err = controller.ListPackages(ctx, logE, false, "test") // Assert - require.NoError(t, err) // Should succeed with empty database - assert.Equal(t, "no packages to display", hook.LastEntry().Message) + if err != nil { + t.Fatal(err) // Should succeed with empty database + } + if diff := cmp.Diff("no packages to display", hook.LastEntry().Message); diff != "" { + t.Errorf("Unexpected log message (-want +got):\n%s", diff) + } }) t.Run("StoreFailed", func(t *testing.T) { @@ -103,7 +124,9 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) testDir, err := afero.TempDir(fs, tempTestDir, "store_failed") - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } days := 1 // Short expiration for testing param := &config.Param{ @@ -117,17 +140,23 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop // We force Keeping the DB open to simulate a failure in the async operation err = controller.TestKeepDBOpen() - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } hook.Reset() for _, pkg := range pkgs { err := controller.StorePackage(logE, pkg.configPkg, pkg.pkgPath) - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } } // Wait for the async operations to complete err = controller.Close(logE) - require.NoError(t, err) // If AsyncStorePackage fails, Close should wait for the async operations to complete, but not return an error + if err != nil { + t.Fatal(err) // If AsyncStorePackage fails, Close should wait for the async operations to complete, but not return an error + } expectedLogMessage := []string{ "store package asynchronously", @@ -139,7 +168,9 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop receivedMessages = append(receivedMessages, entry.Message) } for _, entry := range expectedLogMessage { - assert.Contains(t, receivedMessages, entry) + if !contains(receivedMessages, entry) { + t.Errorf("Expected log message %q not found", entry) + } } }) @@ -149,7 +180,9 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop logE := logrus.NewEntry(logger) // Setup - use a new temp directory for this test testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_store_test") - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } days := 30 param := &config.Param{ @@ -164,22 +197,36 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop // Store the package err = controller.StorePackage(logE, pkgs[0].configPkg, pkgs[0].pkgPath) - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } err = controller.Close(logE) // Close to ensure async operations are completed - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } // List packages - should contain our stored package err = controller.ListPackages(ctx, logE, false, "test") - require.NoError(t, err) - assert.Equal(t, "Test mode: Displaying packages", hook.LastEntry().Message) - assert.Equal(t, 1, hook.LastEntry().Data["TotalPackages"]) - assert.Equal(t, 0, hook.LastEntry().Data["TotalExpired"]) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff("Test mode: Displaying packages", hook.LastEntry().Message); diff != "" { + t.Errorf("Unexpected log message (-want +got):\n%s", diff) + } + if diff := cmp.Diff(1, hook.LastEntry().Data["TotalPackages"]); diff != "" { + t.Errorf("Unexpected total packages (-want +got):\n%s", diff) + } + if diff := cmp.Diff(0, hook.LastEntry().Data["TotalExpired"]); diff != "" { + t.Errorf("Unexpected total expired (-want +got):\n%s", diff) + } hook.Reset() // Verify package was stored correctly lastUsed := controller.GetPackageLastUsed(ctx, logE, pkgs[0].pkgPath) - assert.False(t, lastUsed.IsZero(), "Package should have a last used time") + if lastUsed.IsZero() { + t.Fatal("Package should have a last used time") + } }) t.Run("StoreMultiplePackages", func(t *testing.T) { @@ -187,7 +234,9 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_StoreMultiplePackages_test") - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } days := 30 param := &config.Param{ @@ -203,18 +252,30 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop // Store the package for _, pkg := range pkgs { err := controller.StorePackage(logE, pkg.configPkg, pkg.pkgPath) - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } } err = controller.Close(logE) // Close to ensure async operations are completed - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } // List packages - should contain our stored package err = controller.ListPackages(ctx, logE, false, "test") - require.NoError(t, err) - assert.Equal(t, "Test mode: Displaying packages", hook.LastEntry().Message) - assert.Equal(t, 4, hook.LastEntry().Data["TotalPackages"]) - assert.Equal(t, 0, hook.LastEntry().Data["TotalExpired"]) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff("Test mode: Displaying packages", hook.LastEntry().Message); diff != "" { + t.Errorf("Unexpected log message (-want +got):\n%s", diff) + } + if diff := cmp.Diff(4, hook.LastEntry().Data["TotalPackages"]); diff != "" { + t.Errorf("Unexpected total packages (-want +got):\n%s", diff) + } + if diff := cmp.Diff(0, hook.LastEntry().Data["TotalExpired"]); diff != "" { + t.Errorf("Unexpected total expired (-want +got):\n%s", diff) + } hook.Reset() }) @@ -223,7 +284,9 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_StoreNilPackage_test") - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } days := 30 param := &config.Param{ @@ -234,8 +297,12 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop // Store the package err = controller.StorePackage(logE, nil, tempTestDir) - require.NoError(t, err) - assert.Equal(t, "package is nil, skipping store package", hook.LastEntry().Message) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff("package is nil, skipping store package", hook.LastEntry().Message); diff != "" { + t.Errorf("Unexpected log message (-want +got):\n%s", diff) + } }) t.Run("handleListExpiredPackages - no expired packages", func(t *testing.T) { @@ -243,7 +310,9 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_handle_list_expired") - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } days := 30 param := &config.Param{ @@ -257,15 +326,21 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop err = controller.ListPackages(ctx, logE, true, "test") // Assert - require.NoError(t, err) // Error if no package found - assert.Equal(t, "no packages to display", hook.LastEntry().Message) + if err != nil { + t.Fatal(err) // Error if no package found + } + if diff := cmp.Diff("no packages to display", hook.LastEntry().Message); diff != "" { + t.Errorf("Unexpected log message (-want +got):\n%s", diff) + } }) t.Run("VacuumExpiredPackages workflow", func(t *testing.T) { t.Parallel() logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_expire_test") - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } defer func() { hook.Reset() fs.RemoveAll(testDir) //nolint:errcheck @@ -287,12 +362,16 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop // Create package paths and files for _, pkg := range pkgs { err = fs.MkdirAll(pkg.pkgPath, 0o755) - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } // Create a test file in the package directory testFile := filepath.Join(pkg.pkgPath, "test.txt") err = afero.WriteFile(fs, testFile, []byte("test content"), 0o644) - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } pkgPaths = append(pkgPaths, pkg.pkgPath) } @@ -300,62 +379,95 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop // Store Multiple packages for _, pkg := range pkgs { err = controller.StorePackage(logE, pkg.configPkg, pkg.pkgPath) - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } } // Call Close to ensure all async operations are completed err = controller.Close(logE) - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } // Modify timestamp of one package to be expired oldTime := time.Now().Add(-48 * time.Hour) // 2 days old for _, pkg := range pkgs[:numberPackagesToExpire] { err = controller.SetTimestampPackage(ctx, logE, pkg.configPkg, pkg.pkgPath, oldTime) - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } } // Check Packages after expiration err = controller.ListPackages(ctx, logE, false, "test") - require.NoError(t, err) - assert.Equal(t, numberPackagesToStore, hook.LastEntry().Data["TotalPackages"]) - assert.Equal(t, numberPackagesToExpire, hook.LastEntry().Data["TotalExpired"]) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(numberPackagesToStore, hook.LastEntry().Data["TotalPackages"]); diff != "" { + t.Errorf("Unexpected total packages (-want +got):\n%s", diff) + } + if diff := cmp.Diff(numberPackagesToExpire, hook.LastEntry().Data["TotalExpired"]); diff != "" { + t.Errorf("Unexpected total expired (-want +got):\n%s", diff) + } // List expired packages only err = controller.ListPackages(ctx, logE, true, "test") - require.NoError(t, err) - assert.Equal(t, numberPackagesToExpire, hook.LastEntry().Data["TotalPackages"]) - assert.Equal(t, numberPackagesToExpire, hook.LastEntry().Data["TotalExpired"]) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(numberPackagesToExpire, hook.LastEntry().Data["TotalPackages"]); diff != "" { + t.Errorf("Unexpected total packages (-want +got):\n%s", diff) + } + if diff := cmp.Diff(numberPackagesToExpire, hook.LastEntry().Data["TotalExpired"]); diff != "" { + t.Errorf("Unexpected total expired (-want +got):\n%s", diff) + } // Run vacuum err = controller.Vacuum(ctx, logE) - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } // List expired packages err = controller.ListPackages(ctx, logE, true, "test") - require.NoError(t, err) - assert.Equal(t, "no packages to display", hook.LastEntry().Message) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff("no packages to display", hook.LastEntry().Message); diff != "" { + t.Errorf("Unexpected log message (-want +got):\n%s", diff) + } // Verify Package Paths was removed : for _, pkgPath := range pkgPaths[:numberPackagesToExpire] { exist, err := afero.Exists(fs, pkgPath) - require.NoError(t, err) - assert.False(t, exist, "Package directory should be removed after vacuum") + if err != nil { + t.Fatal(err) + } + if exist { + t.Fatal("Package directory should be removed after vacuum") + } } // Modify timestamp of one package to be expired And lock DB to simulate a failure in the vacuum operation for _, pkg := range pkgs[:numberPackagesToExpire] { err = controller.SetTimestampPackage(ctx, logE, pkg.configPkg, pkg.pkgPath, oldTime) - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } } // Keep Database open to simulate a failure in the vacuum operation err = controller.TestKeepDBOpen() - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } // Run vacuum err = controller.Vacuum(ctx, logE) - require.Error(t, err) - assert.Contains(t, err.Error(), "open database vacuum.db: timeout") + if err == nil || !contains([]string{err.Error()}, "open database vacuum.db: timeout") { + t.Fatalf("Expected timeout error, got %v", err) + } }) t.Run("TestVacuumWithoutExpiredPackages", func(t *testing.T) { @@ -364,7 +476,9 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop logE := logrus.NewEntry(logger) fs := afero.NewOsFs() testDir, err := afero.TempDir(fs, "", "vacuum_no_expired") - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } days := 30 param := &config.Param{ @@ -378,29 +492,50 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop pkgs := generateTestPackages(3, param.RootDir) for _, pkg := range pkgs { err = controller.StorePackage(logE, pkg.configPkg, pkg.pkgPath) - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } } // Call Close to ensure all async operations are completed err = controller.Close(logE) - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } // Run vacuum err = controller.Vacuum(ctx, logE) - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } // Verify no packages were removed err = controller.ListPackages(ctx, logE, false, "test") - require.NoError(t, err) - assert.Equal(t, 3, hook.LastEntry().Data["TotalPackages"]) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(3, hook.LastEntry().Data["TotalPackages"]); diff != "" { + t.Errorf("Unexpected total packages (-want +got):\n%s", diff) + } }) } +func contains(receivedMessages []string, entry string) bool { + for _, msg := range receivedMessages { + if msg == entry { + return true + } + } + return false +} + func TestMockVacuumController_StorePackage(t *testing.T) { t.Parallel() fs := afero.NewOsFs() testDir, err := afero.TempDir(fs, "", "vacuum_no_expired") - require.NoError(t, err) + if err != nil { + t.Fatal(err) + } param := &config.Param{ RootDir: testDir, @@ -436,14 +571,22 @@ func TestMockVacuumController_StorePackage(t *testing.T) { t.Parallel() err := mockCtrl.StorePackage(logE, tt.pkg, tt.pkgPath) if tt.wantErr { - require.Error(t, err) + if err == nil { + t.Errorf("Expected error, got nil") + } } else { - require.NoError(t, err) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } } err = mockCtrl.Vacuum(logE) - require.NoError(t, err) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } err = mockCtrl.Close(logE) - require.NoError(t, err) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } }) } } @@ -455,11 +598,17 @@ func TestNilVacuumController(t *testing.T) { test := generateTestPackages(1, "/tmp") err := mockCtrl.StorePackage(logE, test[0].configPkg, test[0].pkgPath) - require.NoError(t, err) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } err = mockCtrl.Vacuum(logE) - require.NoError(t, err) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } err = mockCtrl.Close(logE) - require.NoError(t, err) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } } type ConfigPackageWithPath struct { From 457e69958618f487b8627eeab295e2d27262e133 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 17:24:12 +0100 Subject: [PATCH 46/73] tests(vacuum): fix lints --- pkg/controller/vacuum/vacuum_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/controller/vacuum/vacuum_test.go b/pkg/controller/vacuum/vacuum_test.go index 51fdb2b5e..b8306698f 100644 --- a/pkg/controller/vacuum/vacuum_test.go +++ b/pkg/controller/vacuum/vacuum_test.go @@ -16,7 +16,7 @@ import ( "github.com/spf13/afero" ) -func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop +func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo t.Parallel() fs := afero.NewOsFs() @@ -109,7 +109,6 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop // Test err = controller.ListPackages(ctx, logE, false, "test") - // Assert if err != nil { t.Fatal(err) // Should succeed with empty database @@ -324,7 +323,6 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop // Test err = controller.ListPackages(ctx, logE, true, "test") - // Assert if err != nil { t.Fatal(err) // Error if no package found From 1b55c9dcf5d389c6ab59c9c61733569c733567be Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 17:39:03 +0100 Subject: [PATCH 47/73] tests(vacuum): add sleep to ensure logs catched before end of test --- pkg/controller/vacuum/vacuum_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/controller/vacuum/vacuum_test.go b/pkg/controller/vacuum/vacuum_test.go index b8306698f..3fa400a85 100644 --- a/pkg/controller/vacuum/vacuum_test.go +++ b/pkg/controller/vacuum/vacuum_test.go @@ -134,7 +134,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo } controller := vacuum.New(context.Background(), param, fs) - numberPackagesToStore := 7 + numberPackagesToStore := 4 pkgs := generateTestPackages(numberPackagesToStore, param.RootDir) // We force Keeping the DB open to simulate a failure in the async operation @@ -157,6 +157,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo t.Fatal(err) // If AsyncStorePackage fails, Close should wait for the async operations to complete, but not return an error } + time.Sleep(1 * time.Second) // Wait for ensure have time to get all logs expectedLogMessage := []string{ "store package asynchronously", "retrying database operation", From 087e6f29beab8c6289a4bee19d83c7b5b7f885e4 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 17:46:36 +0100 Subject: [PATCH 48/73] chore(mod): execute go mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 3032c5d17..b07b4c5c1 100644 --- a/go.mod +++ b/go.mod @@ -78,7 +78,7 @@ require ( github.com/shopspring/decimal v1.4.0 // indirect github.com/sorairolake/lzip-go v0.3.5 // indirect github.com/spf13/cast v1.7.0 // indirect - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.9.0 // indirect github.com/suzuki-shunsuke/go-jsoneq v0.1.2 // indirect github.com/therootcompany/xz v1.0.1 // indirect github.com/ulikunitz/xz v0.5.12 // indirect From ee5e23c5bf837191916f9cd4c1da9b3fc6e0ce2c Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sat, 11 Jan 2025 23:57:16 +0100 Subject: [PATCH 49/73] test(vacuum): add packages to let all async log on fail appears --- pkg/controller/vacuum/vacuum_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/vacuum/vacuum_test.go b/pkg/controller/vacuum/vacuum_test.go index 3fa400a85..175189377 100644 --- a/pkg/controller/vacuum/vacuum_test.go +++ b/pkg/controller/vacuum/vacuum_test.go @@ -134,7 +134,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo } controller := vacuum.New(context.Background(), param, fs) - numberPackagesToStore := 4 + numberPackagesToStore := 7 pkgs := generateTestPackages(numberPackagesToStore, param.RootDir) // We force Keeping the DB open to simulate a failure in the async operation From eb86893d25e51611eac83ea1ff25ce842997f106 Mon Sep 17 00:00:00 2001 From: Nikita COEUR Date: Sun, 12 Jan 2025 08:16:50 +0100 Subject: [PATCH 50/73] test(vacuum): replace afero.TempDir with t.TempDir --- pkg/controller/vacuum/vacuum_test.go | 137 +++++++-------------------- 1 file changed, 33 insertions(+), 104 deletions(-) diff --git a/pkg/controller/vacuum/vacuum_test.go b/pkg/controller/vacuum/vacuum_test.go index 175189377..bd0b8c21c 100644 --- a/pkg/controller/vacuum/vacuum_test.go +++ b/pkg/controller/vacuum/vacuum_test.go @@ -21,26 +21,11 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo fs := afero.NewOsFs() - // Create temp directory for tests - tempTestDir, err := afero.TempDir(fs, "/tmp", "vacuum_test") - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { - err := fs.RemoveAll(tempTestDir) - if err != nil { - t.Fatal(err) - } - }) - t.Run("vacuum disabled", func(t *testing.T) { t.Parallel() logger, _ := test.NewNullLogger() logE := logrus.NewEntry(logger) - testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_disabled") - if err != nil { - t.Fatal(err) - } + testDir := t.TempDir() // Setup param := &config.Param{ RootDir: testDir, @@ -48,7 +33,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo ctx := context.Background() controller := vacuum.New(ctx, param, fs) - err = controller.ListPackages(ctx, logE, false, "test") + err := controller.ListPackages(ctx, logE, false, "test") if err != nil { t.Fatal("Should return nil when vacuum is disabled") } @@ -69,18 +54,15 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) logE.Logger.Level = logrus.DebugLevel - testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_bad_config") - if err != nil { - t.Fatal(err) - } - // Setup + testDir := t.TempDir() + param := &config.Param{ RootDir: testDir, VacuumDays: -1, } controller := vacuum.New(context.Background(), param, fs) - err = controller.StorePackage(logE, nil, testDir) + err := controller.StorePackage(logE, nil, testDir) if err != nil { t.Fatal("Should return nil when vacuum is disabled") } @@ -93,11 +75,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo t.Parallel() logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) - // Setup - use a new temp directory for this test - testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_list_test") - if err != nil { - t.Fatal(err) - } + testDir := t.TempDir() days := 30 param := &config.Param{ @@ -107,11 +85,9 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo ctx := context.Background() controller := vacuum.New(ctx, param, fs) - // Test - err = controller.ListPackages(ctx, logE, false, "test") - // Assert + err := controller.ListPackages(ctx, logE, false, "test") if err != nil { - t.Fatal(err) // Should succeed with empty database + t.Fatal(err) } if diff := cmp.Diff("no packages to display", hook.LastEntry().Message); diff != "" { t.Errorf("Unexpected log message (-want +got):\n%s", diff) @@ -122,10 +98,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo t.Parallel() logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) - testDir, err := afero.TempDir(fs, tempTestDir, "store_failed") - if err != nil { - t.Fatal(err) - } + testDir := t.TempDir() days := 1 // Short expiration for testing param := &config.Param{ @@ -138,7 +111,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo pkgs := generateTestPackages(numberPackagesToStore, param.RootDir) // We force Keeping the DB open to simulate a failure in the async operation - err = controller.TestKeepDBOpen() + err := controller.TestKeepDBOpen() if err != nil { t.Fatal(err) } @@ -178,15 +151,10 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo t.Parallel() logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) - // Setup - use a new temp directory for this test - testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_store_test") - if err != nil { - t.Fatal(err) - } + testDir := t.TempDir() - days := 30 param := &config.Param{ - VacuumDays: days, + VacuumDays: 30, RootDir: testDir, } ctx := context.Background() @@ -196,7 +164,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo pkgs := generateTestPackages(numberPackagesToStore, param.RootDir) // Store the package - err = controller.StorePackage(logE, pkgs[0].configPkg, pkgs[0].pkgPath) + err := controller.StorePackage(logE, pkgs[0].configPkg, pkgs[0].pkgPath) if err != nil { t.Fatal(err) } @@ -233,14 +201,10 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo t.Parallel() logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) - testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_StoreMultiplePackages_test") - if err != nil { - t.Fatal(err) - } + testDir := t.TempDir() - days := 30 param := &config.Param{ - VacuumDays: days, + VacuumDays: 30, RootDir: testDir, } ctx := context.Background() @@ -257,7 +221,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo } } - err = controller.Close(logE) // Close to ensure async operations are completed + err := controller.Close(logE) // Close to ensure async operations are completed if err != nil { t.Fatal(err) } @@ -283,20 +247,16 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo t.Parallel() logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) - testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_StoreNilPackage_test") - if err != nil { - t.Fatal(err) - } + testDir := t.TempDir() - days := 30 param := &config.Param{ - VacuumDays: days, + VacuumDays: 30, RootDir: testDir, } controller := vacuum.New(context.Background(), param, fs) // Store the package - err = controller.StorePackage(logE, nil, tempTestDir) + err := controller.StorePackage(logE, nil, testDir) if err != nil { t.Fatal(err) } @@ -309,21 +269,17 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo t.Parallel() logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) - testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_handle_list_expired") - if err != nil { - t.Fatal(err) - } + testDir := t.TempDir() - days := 30 param := &config.Param{ - VacuumDays: days, + VacuumDays: 30, RootDir: testDir, } ctx := context.Background() controller := vacuum.New(ctx, param, fs) // Test - err = controller.ListPackages(ctx, logE, true, "test") + err := controller.ListPackages(ctx, logE, true, "test") // Assert if err != nil { t.Fatal(err) // Error if no package found @@ -334,20 +290,11 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo }) t.Run("VacuumExpiredPackages workflow", func(t *testing.T) { t.Parallel() + testDir := t.TempDir() logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) - testDir, err := afero.TempDir(fs, tempTestDir, "vacuum_expire_test") - if err != nil { - t.Fatal(err) - } - defer func() { - hook.Reset() - fs.RemoveAll(testDir) //nolint:errcheck - }() - - days := 1 // Short expiration for testing param := &config.Param{ - VacuumDays: days, + VacuumDays: 1, RootDir: testDir, } ctx := context.Background() @@ -360,7 +307,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo // Create package paths and files for _, pkg := range pkgs { - err = fs.MkdirAll(pkg.pkgPath, 0o755) + err := fs.MkdirAll(pkg.pkgPath, 0o755) if err != nil { t.Fatal(err) } @@ -377,14 +324,14 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo // Store Multiple packages for _, pkg := range pkgs { - err = controller.StorePackage(logE, pkg.configPkg, pkg.pkgPath) + err := controller.StorePackage(logE, pkg.configPkg, pkg.pkgPath) if err != nil { t.Fatal(err) } } // Call Close to ensure all async operations are completed - err = controller.Close(logE) + err := controller.Close(logE) if err != nil { t.Fatal(err) } @@ -473,15 +420,10 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo t.Parallel() logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) - fs := afero.NewOsFs() - testDir, err := afero.TempDir(fs, "", "vacuum_no_expired") - if err != nil { - t.Fatal(err) - } + testDir := t.TempDir() - days := 30 param := &config.Param{ - VacuumDays: days, + VacuumDays: 30, RootDir: testDir, } ctx := context.Background() @@ -490,14 +432,14 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo // Store non-expired packages pkgs := generateTestPackages(3, param.RootDir) for _, pkg := range pkgs { - err = controller.StorePackage(logE, pkg.configPkg, pkg.pkgPath) + err := controller.StorePackage(logE, pkg.configPkg, pkg.pkgPath) if err != nil { t.Fatal(err) } } // Call Close to ensure all async operations are completed - err = controller.Close(logE) + err := controller.Close(logE) if err != nil { t.Fatal(err) } @@ -530,11 +472,7 @@ func contains(receivedMessages []string, entry string) bool { func TestMockVacuumController_StorePackage(t *testing.T) { t.Parallel() - fs := afero.NewOsFs() - testDir, err := afero.TempDir(fs, "", "vacuum_no_expired") - if err != nil { - t.Fatal(err) - } + testDir := t.TempDir() param := &config.Param{ RootDir: testDir, @@ -658,19 +596,10 @@ func benchmarkVacuumStorePackages(b *testing.B, pkgCount int) { logE := logrus.NewEntry(logrus.New()) fs := afero.NewOsFs() - // Benchmark sync configuration - syncf, errf := afero.TempDir(fs, "/tmp", "vacuum_test_sync") - if errf != nil { - b.Fatal(errf) - } + syncf := b.TempDir() pkgs := generateTestPackages(pkgCount, syncf) vacuumDays := 5 syncParam := &config.Param{RootDir: syncf, VacuumDays: vacuumDays} - defer func() { - if err := fs.RemoveAll(syncf); err != nil { - b.Fatal(err) - } - }() b.Run("Sync", func(b *testing.B) { controller := vacuum.New(context.Background(), syncParam, fs) From 7721951b06605636e3ddbc9106f87ae0c9312941 Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Mon, 20 Jan 2025 21:54:01 +0900 Subject: [PATCH 51/73] refactor: minimize the scope of err variables --- pkg/controller/vacuum/vacuum_test.go | 161 +++++++++------------------ 1 file changed, 54 insertions(+), 107 deletions(-) diff --git a/pkg/controller/vacuum/vacuum_test.go b/pkg/controller/vacuum/vacuum_test.go index bd0b8c21c..38682f6b3 100644 --- a/pkg/controller/vacuum/vacuum_test.go +++ b/pkg/controller/vacuum/vacuum_test.go @@ -25,27 +25,23 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo t.Parallel() logger, _ := test.NewNullLogger() logE := logrus.NewEntry(logger) - testDir := t.TempDir() // Setup param := &config.Param{ - RootDir: testDir, + RootDir: t.TempDir(), } ctx := context.Background() controller := vacuum.New(ctx, param, fs) - err := controller.ListPackages(ctx, logE, false, "test") - if err != nil { - t.Fatal("Should return nil when vacuum is disabled") + if err := controller.ListPackages(ctx, logE, false, "test"); err != nil { + t.Fatalf("should return nil when vacuum is disabled: %v", err) } - err = controller.Vacuum(ctx, logE) - if err != nil { - t.Fatal("Should return nil when vacuum is disabled") + if err := controller.Vacuum(ctx, logE); err != nil { + t.Fatalf("should return nil when vacuum is disabled: %v", err) } - err = controller.Close(logE) - if err != nil { - t.Fatal("Should return nil when vacuum is disabled") + if err := controller.Close(logE); err != nil { + t.Fatalf("should return nil when vacuum is disabled: %v", err) } }) @@ -62,9 +58,8 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo } controller := vacuum.New(context.Background(), param, fs) - err := controller.StorePackage(logE, nil, testDir) - if err != nil { - t.Fatal("Should return nil when vacuum is disabled") + if err := controller.StorePackage(logE, nil, testDir); err != nil { + t.Fatalf("should return nil when vacuum is disabled: %v", err) } if diff := cmp.Diff("vacuum is disabled. AQUA_VACUUM_DAYS is not set or invalid.", hook.LastEntry().Message); diff != "" { t.Errorf("Unexpected log message (-want +got):\n%s", diff) @@ -75,18 +70,14 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo t.Parallel() logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) - testDir := t.TempDir() - - days := 30 param := &config.Param{ - VacuumDays: days, - RootDir: testDir, + VacuumDays: 30, + RootDir: t.TempDir(), } ctx := context.Background() controller := vacuum.New(ctx, param, fs) - err := controller.ListPackages(ctx, logE, false, "test") - if err != nil { + if err := controller.ListPackages(ctx, logE, false, "test"); err != nil { t.Fatal(err) } if diff := cmp.Diff("no packages to display", hook.LastEntry().Message); diff != "" { @@ -98,12 +89,9 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo t.Parallel() logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) - testDir := t.TempDir() - - days := 1 // Short expiration for testing param := &config.Param{ - VacuumDays: days, - RootDir: testDir, + VacuumDays: 1, // Short expiration for testing + RootDir: t.TempDir(), } controller := vacuum.New(context.Background(), param, fs) @@ -111,22 +99,19 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo pkgs := generateTestPackages(numberPackagesToStore, param.RootDir) // We force Keeping the DB open to simulate a failure in the async operation - err := controller.TestKeepDBOpen() - if err != nil { + if err := controller.TestKeepDBOpen(); err != nil { t.Fatal(err) } hook.Reset() for _, pkg := range pkgs { - err := controller.StorePackage(logE, pkg.configPkg, pkg.pkgPath) - if err != nil { + if err := controller.StorePackage(logE, pkg.configPkg, pkg.pkgPath); err != nil { t.Fatal(err) } } // Wait for the async operations to complete - err = controller.Close(logE) - if err != nil { + if err := controller.Close(logE); err != nil { t.Fatal(err) // If AsyncStorePackage fails, Close should wait for the async operations to complete, but not return an error } @@ -151,11 +136,9 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo t.Parallel() logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) - testDir := t.TempDir() - param := &config.Param{ VacuumDays: 30, - RootDir: testDir, + RootDir: t.TempDir(), } ctx := context.Background() controller := vacuum.New(ctx, param, fs) @@ -164,19 +147,16 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo pkgs := generateTestPackages(numberPackagesToStore, param.RootDir) // Store the package - err := controller.StorePackage(logE, pkgs[0].configPkg, pkgs[0].pkgPath) - if err != nil { + if err := controller.StorePackage(logE, pkgs[0].configPkg, pkgs[0].pkgPath); err != nil { t.Fatal(err) } - - err = controller.Close(logE) // Close to ensure async operations are completed - if err != nil { + // Close to ensure async operations are completed + if err := controller.Close(logE); err != nil { t.Fatal(err) } // List packages - should contain our stored package - err = controller.ListPackages(ctx, logE, false, "test") - if err != nil { + if err := controller.ListPackages(ctx, logE, false, "test"); err != nil { t.Fatal(err) } if diff := cmp.Diff("Test mode: Displaying packages", hook.LastEntry().Message); diff != "" { @@ -201,11 +181,9 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo t.Parallel() logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) - testDir := t.TempDir() - param := &config.Param{ VacuumDays: 30, - RootDir: testDir, + RootDir: t.TempDir(), } ctx := context.Background() controller := vacuum.New(ctx, param, fs) @@ -221,14 +199,13 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo } } - err := controller.Close(logE) // Close to ensure async operations are completed - if err != nil { + // Close to ensure async operations are completed + if err := controller.Close(logE); err != nil { t.Fatal(err) } // List packages - should contain our stored package - err = controller.ListPackages(ctx, logE, false, "test") - if err != nil { + if err := controller.ListPackages(ctx, logE, false, "test"); err != nil { t.Fatal(err) } if diff := cmp.Diff("Test mode: Displaying packages", hook.LastEntry().Message); diff != "" { @@ -256,8 +233,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo controller := vacuum.New(context.Background(), param, fs) // Store the package - err := controller.StorePackage(logE, nil, testDir) - if err != nil { + if err := controller.StorePackage(logE, nil, testDir); err != nil { t.Fatal(err) } if diff := cmp.Diff("package is nil, skipping store package", hook.LastEntry().Message); diff != "" { @@ -269,19 +245,14 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo t.Parallel() logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) - testDir := t.TempDir() - param := &config.Param{ VacuumDays: 30, - RootDir: testDir, + RootDir: t.TempDir(), } ctx := context.Background() controller := vacuum.New(ctx, param, fs) - // Test - err := controller.ListPackages(ctx, logE, true, "test") - // Assert - if err != nil { + if err := controller.ListPackages(ctx, logE, true, "test"); err != nil { t.Fatal(err) // Error if no package found } if diff := cmp.Diff("no packages to display", hook.LastEntry().Message); diff != "" { @@ -290,12 +261,11 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo }) t.Run("VacuumExpiredPackages workflow", func(t *testing.T) { t.Parallel() - testDir := t.TempDir() logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) param := &config.Param{ VacuumDays: 1, - RootDir: testDir, + RootDir: t.TempDir(), } ctx := context.Background() controller := vacuum.New(ctx, param, fs) @@ -307,15 +277,13 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo // Create package paths and files for _, pkg := range pkgs { - err := fs.MkdirAll(pkg.pkgPath, 0o755) - if err != nil { + if err := fs.MkdirAll(pkg.pkgPath, 0o755); err != nil { t.Fatal(err) } // Create a test file in the package directory testFile := filepath.Join(pkg.pkgPath, "test.txt") - err = afero.WriteFile(fs, testFile, []byte("test content"), 0o644) - if err != nil { + if err := afero.WriteFile(fs, testFile, []byte("test content"), 0o644); err != nil { t.Fatal(err) } @@ -324,30 +292,26 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo // Store Multiple packages for _, pkg := range pkgs { - err := controller.StorePackage(logE, pkg.configPkg, pkg.pkgPath) - if err != nil { + if err := controller.StorePackage(logE, pkg.configPkg, pkg.pkgPath); err != nil { t.Fatal(err) } } // Call Close to ensure all async operations are completed - err := controller.Close(logE) - if err != nil { + if err := controller.Close(logE); err != nil { t.Fatal(err) } // Modify timestamp of one package to be expired oldTime := time.Now().Add(-48 * time.Hour) // 2 days old for _, pkg := range pkgs[:numberPackagesToExpire] { - err = controller.SetTimestampPackage(ctx, logE, pkg.configPkg, pkg.pkgPath, oldTime) - if err != nil { + if err := controller.SetTimestampPackage(ctx, logE, pkg.configPkg, pkg.pkgPath, oldTime); err != nil { t.Fatal(err) } } // Check Packages after expiration - err = controller.ListPackages(ctx, logE, false, "test") - if err != nil { + if err := controller.ListPackages(ctx, logE, false, "test"); err != nil { t.Fatal(err) } if diff := cmp.Diff(numberPackagesToStore, hook.LastEntry().Data["TotalPackages"]); diff != "" { @@ -358,8 +322,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo } // List expired packages only - err = controller.ListPackages(ctx, logE, true, "test") - if err != nil { + if err := controller.ListPackages(ctx, logE, true, "test"); err != nil { t.Fatal(err) } if diff := cmp.Diff(numberPackagesToExpire, hook.LastEntry().Data["TotalPackages"]); diff != "" { @@ -370,14 +333,12 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo } // Run vacuum - err = controller.Vacuum(ctx, logE) - if err != nil { + if err := controller.Vacuum(ctx, logE); err != nil { t.Fatal(err) } // List expired packages - err = controller.ListPackages(ctx, logE, true, "test") - if err != nil { + if err := controller.ListPackages(ctx, logE, true, "test"); err != nil { t.Fatal(err) } if diff := cmp.Diff("no packages to display", hook.LastEntry().Message); diff != "" { @@ -397,21 +358,18 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo // Modify timestamp of one package to be expired And lock DB to simulate a failure in the vacuum operation for _, pkg := range pkgs[:numberPackagesToExpire] { - err = controller.SetTimestampPackage(ctx, logE, pkg.configPkg, pkg.pkgPath, oldTime) - if err != nil { + if err := controller.SetTimestampPackage(ctx, logE, pkg.configPkg, pkg.pkgPath, oldTime); err != nil { t.Fatal(err) } } // Keep Database open to simulate a failure in the vacuum operation - err = controller.TestKeepDBOpen() - if err != nil { + if err := controller.TestKeepDBOpen(); err != nil { t.Fatal(err) } // Run vacuum - err = controller.Vacuum(ctx, logE) - if err == nil || !contains([]string{err.Error()}, "open database vacuum.db: timeout") { + if err := controller.Vacuum(ctx, logE); err == nil || !contains([]string{err.Error()}, "open database vacuum.db: timeout") { t.Fatalf("Expected timeout error, got %v", err) } }) @@ -420,11 +378,9 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo t.Parallel() logger, hook := test.NewNullLogger() logE := logrus.NewEntry(logger) - testDir := t.TempDir() - param := &config.Param{ VacuumDays: 30, - RootDir: testDir, + RootDir: t.TempDir(), } ctx := context.Background() controller := vacuum.New(ctx, param, fs) @@ -439,20 +395,17 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo } // Call Close to ensure all async operations are completed - err := controller.Close(logE) - if err != nil { + if err := controller.Close(logE); err != nil { t.Fatal(err) } // Run vacuum - err = controller.Vacuum(ctx, logE) - if err != nil { + if err := controller.Vacuum(ctx, logE); err != nil { t.Fatal(err) } // Verify no packages were removed - err = controller.ListPackages(ctx, logE, false, "test") - if err != nil { + if err := controller.ListPackages(ctx, logE, false, "test"); err != nil { t.Fatal(err) } if diff := cmp.Diff(3, hook.LastEntry().Data["TotalPackages"]); diff != "" { @@ -506,8 +459,7 @@ func TestMockVacuumController_StorePackage(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - err := mockCtrl.StorePackage(logE, tt.pkg, tt.pkgPath) - if tt.wantErr { + if err := mockCtrl.StorePackage(logE, tt.pkg, tt.pkgPath); tt.wantErr { if err == nil { t.Errorf("Expected error, got nil") } @@ -516,12 +468,11 @@ func TestMockVacuumController_StorePackage(t *testing.T) { t.Errorf("Unexpected error: %v", err) } } - err = mockCtrl.Vacuum(logE) - if err != nil { + if err := mockCtrl.Vacuum(logE); err != nil { t.Errorf("Unexpected error: %v", err) } - err = mockCtrl.Close(logE) - if err != nil { + + if err := mockCtrl.Close(logE); err != nil { t.Errorf("Unexpected error: %v", err) } }) @@ -534,16 +485,13 @@ func TestNilVacuumController(t *testing.T) { mockCtrl := &vacuum.NilVacuumController{} test := generateTestPackages(1, "/tmp") - err := mockCtrl.StorePackage(logE, test[0].configPkg, test[0].pkgPath) - if err != nil { + if err := mockCtrl.StorePackage(logE, test[0].configPkg, test[0].pkgPath); err != nil { t.Errorf("Unexpected error: %v", err) } - err = mockCtrl.Vacuum(logE) - if err != nil { + if err := mockCtrl.Vacuum(logE); err != nil { t.Errorf("Unexpected error: %v", err) } - err = mockCtrl.Close(logE) - if err != nil { + if err := mockCtrl.Close(logE); err != nil { t.Errorf("Unexpected error: %v", err) } } @@ -598,8 +546,7 @@ func benchmarkVacuumStorePackages(b *testing.B, pkgCount int) { syncf := b.TempDir() pkgs := generateTestPackages(pkgCount, syncf) - vacuumDays := 5 - syncParam := &config.Param{RootDir: syncf, VacuumDays: vacuumDays} + syncParam := &config.Param{RootDir: syncf, VacuumDays: 5} b.Run("Sync", func(b *testing.B) { controller := vacuum.New(context.Background(), syncParam, fs) From 4fa64102c031994dc7f7b71ad42978b7430883bd Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Mon, 20 Jan 2025 21:59:23 +0900 Subject: [PATCH 52/73] refactor: replace contains with strings.HasPrefix --- pkg/controller/vacuum/vacuum_internal_test.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/pkg/controller/vacuum/vacuum_internal_test.go b/pkg/controller/vacuum/vacuum_internal_test.go index f17a0dd6e..d00ada2cb 100644 --- a/pkg/controller/vacuum/vacuum_internal_test.go +++ b/pkg/controller/vacuum/vacuum_internal_test.go @@ -3,6 +3,7 @@ package vacuum import ( "context" "encoding/json" + "strings" "testing" "time" @@ -44,8 +45,7 @@ func TestEncodePackageEntry(t *testing.T) { } var decodedEntry PackageEntry - err = json.Unmarshal(data, &decodedEntry) - if err != nil { + if err = json.Unmarshal(data, &decodedEntry); err != nil { t.Fatalf("unexpected error: %v", err) } if diff := cmp.Diff(pkgEntry.LastUsageTime.Unix(), decodedEntry.LastUsageTime.Unix()); diff != "" { @@ -89,11 +89,7 @@ func TestDecodePackageEntry_Error(t *testing.T) { if err == nil { t.Fatalf("expected an error but got nil") } - if diff := cmp.Diff(true, contains(err.Error(), "unmarshal package entry")); diff != "" { + if diff := cmp.Diff(true, strings.HasPrefix(err.Error(), "unmarshal package entry")); diff != "" { t.Errorf("unexpected error message (-want +got):\n%s", diff) } } - -func contains(s, substr string) bool { - return len(s) >= len(substr) && s[:len(substr)] == substr -} From ec0a9871eaf370cf9acf111d7f30781cef1cae95 Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 00:45:21 +0900 Subject: [PATCH 53/73] refactor: remove generatePackageKey --- pkg/config/package.go | 20 +++++++------- pkg/config/package_test.go | 11 ++++---- pkg/controller/vacuum/vacuum.go | 14 +--------- pkg/controller/vacuum/vacuum_test.go | 39 +++++++++++++--------------- pkg/installpackage/check_file.go | 3 ++- pkg/installpackage/installer.go | 2 +- pkg/installpackage/link.go | 8 +++--- pkg/installpackage/proxy.go | 3 ++- 8 files changed, 43 insertions(+), 57 deletions(-) diff --git a/pkg/config/package.go b/pkg/config/package.go index 630ce6a35..4db81a87b 100644 --- a/pkg/config/package.go +++ b/pkg/config/package.go @@ -31,7 +31,7 @@ func (p *Package) ExePath(rootDir string, file *registry.File, rt *runtime.Runti return filepath.Join(rootDir, "pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, p.Package.Version, "bin", file.Name), nil } - pkgPath, err := p.PkgPath(rootDir, rt) + pkgPath, err := p.PkgPath(rt) if err != nil { return "", err } @@ -39,7 +39,7 @@ func (p *Package) ExePath(rootDir string, file *registry.File, rt *runtime.Runti if err != nil { return "", fmt.Errorf("get a file path: %w", err) } - return filepath.Join(pkgPath, fileSrc), nil + return filepath.Join(rootDir, pkgPath, fileSrc), nil } func (p *Package) RenderAsset(rt *runtime.Runtime) (string, error) { @@ -80,7 +80,7 @@ func (p *Package) RenderPath() (string, error) { return p.RenderTemplateString(pkgInfo.GetPath(), &runtime.Runtime{}) } -func (p *Package) PkgPath(rootDir string, rt *runtime.Runtime) (string, error) { //nolint:cyclop +func (p *Package) PkgPath(rt *runtime.Runtime) (string, error) { //nolint:cyclop pkgInfo := p.PackageInfo pkg := p.Package assetName, err := p.RenderAsset(rt) @@ -89,23 +89,23 @@ func (p *Package) PkgPath(rootDir string, rt *runtime.Runtime) (string, error) { } switch pkgInfo.Type { case PkgInfoTypeGitHubArchive: - return filepath.Join(rootDir, "pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version), nil + return filepath.Join("pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version), nil case PkgInfoTypeGoBuild: - return filepath.Join(rootDir, "pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version, "src"), nil + return filepath.Join("pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version, "src"), nil case PkgInfoTypeGoInstall: p, err := p.RenderPath() if err != nil { return "", fmt.Errorf("render Go Module Path: %w", err) } - return filepath.Join(rootDir, "pkgs", pkgInfo.Type, p, pkg.Version, "bin"), nil + return filepath.Join("pkgs", pkgInfo.Type, p, pkg.Version, "bin"), nil case PkgInfoTypeCargo: registry := "crates.io" - return filepath.Join(rootDir, "pkgs", pkgInfo.Type, registry, pkgInfo.Crate, strings.TrimPrefix(pkg.Version, "v")), nil + return filepath.Join("pkgs", pkgInfo.Type, registry, pkgInfo.Crate, strings.TrimPrefix(pkg.Version, "v")), nil case PkgInfoTypeGitHubContent, PkgInfoTypeGitHubRelease: if pkgInfo.RepoOwner == "aquaproj" && (pkgInfo.RepoName == "aqua" || pkgInfo.RepoName == "aqua-proxy") { - return filepath.Join(rootDir, "internal", "pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version, assetName), nil + return filepath.Join("internal", "pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version, assetName), nil } - return filepath.Join(rootDir, "pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version, assetName), nil + return filepath.Join("pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version, assetName), nil case PkgInfoTypeHTTP: uS, err := p.RenderURL(rt) if err != nil { @@ -115,7 +115,7 @@ func (p *Package) PkgPath(rootDir string, rt *runtime.Runtime) (string, error) { if err != nil { return "", fmt.Errorf("parse the URL: %w", err) } - return filepath.Join(rootDir, "pkgs", pkgInfo.Type, u.Host, u.Path), nil + return filepath.Join("pkgs", pkgInfo.Type, u.Host, u.Path), nil } return "", nil } diff --git a/pkg/config/package_test.go b/pkg/config/package_test.go index 00af2804e..0b109c4d1 100644 --- a/pkg/config/package_test.go +++ b/pkg/config/package_test.go @@ -140,7 +140,6 @@ func TestPackage_RenderAsset(t *testing.T) { //nolint:funlen func TestPackageInfo_PkgPath(t *testing.T) { //nolint:funlen t.Parallel() - rootDir := "/tmp/aqua" data := []struct { title string exp string @@ -148,7 +147,7 @@ func TestPackageInfo_PkgPath(t *testing.T) { //nolint:funlen }{ { title: "github_archive", - exp: "/tmp/aqua/pkgs/github_archive/github.com/tfutils/tfenv/v2.2.2", + exp: "pkgs/github_archive/github.com/tfutils/tfenv/v2.2.2", pkg: &config.Package{ PackageInfo: ®istry.PackageInfo{ Type: "github_archive", @@ -162,7 +161,7 @@ func TestPackageInfo_PkgPath(t *testing.T) { //nolint:funlen }, { title: "github_content", - exp: "/tmp/aqua/pkgs/github_content/github.com/aquaproj/aqua-installer/v0.2.0/aqua-installer", + exp: "pkgs/github_content/github.com/aquaproj/aqua-installer/v0.2.0/aqua-installer", pkg: &config.Package{ PackageInfo: ®istry.PackageInfo{ Type: "github_content", @@ -177,7 +176,7 @@ func TestPackageInfo_PkgPath(t *testing.T) { //nolint:funlen }, { title: "github_release", - exp: "/tmp/aqua/pkgs/github_release/github.com/suzuki-shunsuke/ci-info/v0.7.7/ci-info.tar.gz", + exp: "pkgs/github_release/github.com/suzuki-shunsuke/ci-info/v0.7.7/ci-info.tar.gz", pkg: &config.Package{ PackageInfo: ®istry.PackageInfo{ Type: "github_release", @@ -193,7 +192,7 @@ func TestPackageInfo_PkgPath(t *testing.T) { //nolint:funlen }, { title: "http", - exp: "/tmp/aqua/pkgs/http/example.com/foo-1.0.0.zip", + exp: "pkgs/http/example.com/foo-1.0.0.zip", pkg: &config.Package{ PackageInfo: ®istry.PackageInfo{ Type: "http", @@ -210,7 +209,7 @@ func TestPackageInfo_PkgPath(t *testing.T) { //nolint:funlen for _, d := range data { t.Run(d.title, func(t *testing.T) { t.Parallel() - pkgPath, err := d.pkg.PkgPath(rootDir, rt) + pkgPath, err := d.pkg.PkgPath(rt) if err != nil { t.Fatal(err) } diff --git a/pkg/controller/vacuum/vacuum.go b/pkg/controller/vacuum/vacuum.go index ff80891fb..7ff58e48b 100644 --- a/pkg/controller/vacuum/vacuum.go +++ b/pkg/controller/vacuum/vacuum.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "path/filepath" - "strings" "sync" "time" @@ -101,7 +100,6 @@ func (vc *Controller) StorePackage(logE *logrus.Entry, pkg *config.Package, pkgP // getVacuumPackage converts a config func (vc *Controller) getVacuumPackage(configPkg *config.Package, pkgPath string) *Package { - pkgPath = generatePackageKey(vc.Param.RootDir, pkgPath) return &Package{ Type: configPkg.PackageInfo.Type, Name: configPkg.Package.Name, @@ -110,13 +108,6 @@ func (vc *Controller) getVacuumPackage(configPkg *config.Package, pkgPath string } } -// generatePackageKey generates a package key based on the root directory and package path. -func generatePackageKey(rootDir string, pkgPath string) string { - const splitParts = 2 - pkgPath = strings.SplitN(pkgPath, rootDir+"/pkgs/", splitParts)[1] - return pkgPath -} - // handleAsyncStorePackage processes a list of configuration packages asynchronously. func (vc *Controller) handleAsyncStorePackage(logE *logrus.Entry, vacuumPkg *Package) error { if vacuumPkg == nil { @@ -436,9 +427,7 @@ func (vc *Controller) removePackages(ctx context.Context, logE *logrus.Entry, pk // removePackageVersionPath removes the specified package version directory and its parent directory if it becomes empty. func (vc *Controller) removePackageVersionPath(param *config.Param, path string) error { - pkgsPath := filepath.Join(param.RootDir, "pkgs") - pkgVersionPath := filepath.Join(pkgsPath, path) - if err := vc.fs.RemoveAll(pkgVersionPath); err != nil { + if err := vc.fs.RemoveAll(filepath.Join(param.RootDir, path)); err != nil { return fmt.Errorf("remove package version directories: %w", err) } return nil @@ -481,7 +470,6 @@ func (vc *Controller) SetTimestampPackage(ctx context.Context, logE *logrus.Entr // retrievePackageEntry retrieves a package entry from the database by key. for testing purposes. func (vc *Controller) retrievePackageEntry(ctx context.Context, logE *logrus.Entry, key string) (*PackageEntry, error) { var pkgEntry *PackageEntry - key = generatePackageKey(vc.Param.RootDir, key) err := vc.withDBRetry(ctx, logE, func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(bucketNamePkgs)) if b == nil { diff --git a/pkg/controller/vacuum/vacuum_test.go b/pkg/controller/vacuum/vacuum_test.go index 38682f6b3..e376e3558 100644 --- a/pkg/controller/vacuum/vacuum_test.go +++ b/pkg/controller/vacuum/vacuum_test.go @@ -96,7 +96,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo controller := vacuum.New(context.Background(), param, fs) numberPackagesToStore := 7 - pkgs := generateTestPackages(numberPackagesToStore, param.RootDir) + pkgs := generateTestPackages(numberPackagesToStore) // We force Keeping the DB open to simulate a failure in the async operation if err := controller.TestKeepDBOpen(); err != nil { @@ -144,7 +144,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo controller := vacuum.New(ctx, param, fs) numberPackagesToStore := 1 - pkgs := generateTestPackages(numberPackagesToStore, param.RootDir) + pkgs := generateTestPackages(numberPackagesToStore) // Store the package if err := controller.StorePackage(logE, pkgs[0].configPkg, pkgs[0].pkgPath); err != nil { @@ -189,7 +189,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo controller := vacuum.New(ctx, param, fs) numberPackagesToStore := 4 - pkgs := generateTestPackages(numberPackagesToStore, param.RootDir) + pkgs := generateTestPackages(numberPackagesToStore) // Store the package for _, pkg := range pkgs { @@ -272,22 +272,23 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo numberPackagesToStore := 3 numberPackagesToExpire := 1 - pkgs := generateTestPackages(numberPackagesToStore, param.RootDir) + pkgs := generateTestPackages(numberPackagesToStore) pkgPaths := make([]string, 0, numberPackagesToStore) // Create package paths and files for _, pkg := range pkgs { - if err := fs.MkdirAll(pkg.pkgPath, 0o755); err != nil { + pkgPath := filepath.Join(param.RootDir, pkg.pkgPath) + if err := fs.MkdirAll(pkgPath, 0o755); err != nil { t.Fatal(err) } // Create a test file in the package directory - testFile := filepath.Join(pkg.pkgPath, "test.txt") + testFile := filepath.Join(pkgPath, "test.txt") if err := afero.WriteFile(fs, testFile, []byte("test content"), 0o644); err != nil { t.Fatal(err) } - pkgPaths = append(pkgPaths, pkg.pkgPath) + pkgPaths = append(pkgPaths, pkgPath) } // Store Multiple packages @@ -386,7 +387,7 @@ func TestVacuum(t *testing.T) { //nolint:funlen,maintidx,cyclop,gocognit,gocyclo controller := vacuum.New(ctx, param, fs) // Store non-expired packages - pkgs := generateTestPackages(3, param.RootDir) + pkgs := generateTestPackages(3) for _, pkg := range pkgs { err := controller.StorePackage(logE, pkg.configPkg, pkg.pkgPath) if err != nil { @@ -425,16 +426,10 @@ func contains(receivedMessages []string, entry string) bool { func TestMockVacuumController_StorePackage(t *testing.T) { t.Parallel() - testDir := t.TempDir() - - param := &config.Param{ - RootDir: testDir, - } - logE := logrus.NewEntry(logrus.New()) mockCtrl := vacuum.NewMockVacuumController() - pkgs := generateTestPackages(2, param.RootDir) + pkgs := generateTestPackages(2) tests := []struct { name string @@ -484,7 +479,7 @@ func TestNilVacuumController(t *testing.T) { logE := logrus.NewEntry(logrus.New()) mockCtrl := &vacuum.NilVacuumController{} - test := generateTestPackages(1, "/tmp") + test := generateTestPackages(1) if err := mockCtrl.StorePackage(logE, test[0].configPkg, test[0].pkgPath); err != nil { t.Errorf("Unexpected error: %v", err) } @@ -501,7 +496,7 @@ type ConfigPackageWithPath struct { pkgPath string } -func generateTestPackages(count int, rootDir string) []ConfigPackageWithPath { +func generateTestPackages(count int) []ConfigPackageWithPath { pkgs := make([]ConfigPackageWithPath, count) for i := range pkgs { pkgType := "github_release" @@ -522,7 +517,7 @@ func generateTestPackages(count int, rootDir string) []ConfigPackageWithPath { Asset: asset, }, }, - pkgPath: filepath.Join(rootDir, "pkgs", pkgType, "github.com", pkgName, asset), + pkgPath: filepath.Join("pkgs", pkgType, "github.com", pkgName, asset), } } return pkgs @@ -544,9 +539,11 @@ func benchmarkVacuumStorePackages(b *testing.B, pkgCount int) { logE := logrus.NewEntry(logrus.New()) fs := afero.NewOsFs() - syncf := b.TempDir() - pkgs := generateTestPackages(pkgCount, syncf) - syncParam := &config.Param{RootDir: syncf, VacuumDays: 5} + pkgs := generateTestPackages(pkgCount) + syncParam := &config.Param{ + RootDir: b.TempDir(), + VacuumDays: 5, + } b.Run("Sync", func(b *testing.B) { controller := vacuum.New(context.Background(), syncParam, fs) diff --git a/pkg/installpackage/check_file.go b/pkg/installpackage/check_file.go index 19afec278..766b9106b 100644 --- a/pkg/installpackage/check_file.go +++ b/pkg/installpackage/check_file.go @@ -109,10 +109,11 @@ func (is *Installer) checkFileSrc(ctx context.Context, logE *logrus.Entry, pkg * return is.checkFileSrcGo(ctx, logE, pkg, file) } - pkgPath, err := pkg.PkgPath(is.rootDir, is.runtime) + pkgPath, err := pkg.PkgPath(is.runtime) if err != nil { return "", fmt.Errorf("get the package install path: %w", err) } + pkgPath = filepath.Join(is.rootDir, pkgPath) fileSrc, err := pkg.RenameFile(logE, is.fs, pkgPath, file, is.runtime) if err != nil { diff --git a/pkg/installpackage/installer.go b/pkg/installpackage/installer.go index 928656d94..e166a8000 100644 --- a/pkg/installpackage/installer.go +++ b/pkg/installpackage/installer.go @@ -278,7 +278,7 @@ func (is *Installer) InstallPackage(ctx context.Context, logE *logrus.Entry, par return fmt.Errorf("render the asset name: %w", err) } - pkgPath, err := pkg.PkgPath(is.rootDir, is.runtime) + pkgPath, err := pkg.PkgPath(is.runtime) if err != nil { return fmt.Errorf("get the package install path: %w", err) } diff --git a/pkg/installpackage/link.go b/pkg/installpackage/link.go index b3a7be76e..2c7c019f5 100644 --- a/pkg/installpackage/link.go +++ b/pkg/installpackage/link.go @@ -19,12 +19,12 @@ func (is *Installer) createLinks(logE *logrus.Entry, pkgs []*config.Package) boo var aquaProxyPathOnWindows string if is.runtime.IsWindows() { pkg := proxyPkg() - pkgPath, err := pkg.PkgPath(is.rootDir, is.runtime) + pkgPath, err := pkg.PkgPath(is.runtime) if err != nil { logerr.WithError(logE, err).Error("get a path to aqua-proxy") failed = true } - aquaProxyPathOnWindows = filepath.Join(pkgPath, "aqua-proxy.exe") + aquaProxyPathOnWindows = filepath.Join(is.rootDir, pkgPath, "aqua-proxy.exe") } for _, pkg := range pkgs { @@ -116,11 +116,11 @@ func (is *Installer) recreateHardLinks() error { } pkg := proxyPkg() - pkgPath, err := pkg.PkgPath(is.rootDir, is.runtime) + pkgPath, err := pkg.PkgPath(is.runtime) if err != nil { return err //nolint:wrapcheck } - a := filepath.Join(pkgPath, "aqua-proxy.exe") + a := filepath.Join(is.rootDir, pkgPath, "aqua-proxy.exe") for _, info := range infos { if info.Name() == "aqua.exe" { diff --git a/pkg/installpackage/proxy.go b/pkg/installpackage/proxy.go index 78321edb0..239a543a1 100644 --- a/pkg/installpackage/proxy.go +++ b/pkg/installpackage/proxy.go @@ -59,10 +59,11 @@ func (is *Installer) InstallProxy(ctx context.Context, logE *logrus.Entry) error return err //nolint:wrapcheck } - pkgPath, err := pkg.PkgPath(is.rootDir, is.runtime) + pkgPath, err := pkg.PkgPath(is.runtime) if err != nil { return err //nolint:wrapcheck } + pkgPath = filepath.Join(is.rootDir, pkgPath) // create a symbolic link binName := proxyName From eda0509d17a83926f0e07625faa61b6153a9121e Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 00:58:18 +0900 Subject: [PATCH 54/73] fix: fix pkgPath --- pkg/installpackage/installer.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/installpackage/installer.go b/pkg/installpackage/installer.go index e166a8000..2dfc06fb2 100644 --- a/pkg/installpackage/installer.go +++ b/pkg/installpackage/installer.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "path/filepath" "sync" "github.com/aquaproj/aqua/v2/pkg/checksum" @@ -286,6 +287,7 @@ func (is *Installer) InstallPackage(ctx context.Context, logE *logrus.Entry, par if err := is.vacuum.StorePackage(logE, pkg, pkgPath); err != nil { logerr.WithError(logE, err).Error("store the package") } + pkgPath = filepath.Join(is.rootDir, pkgPath) if err := is.downloadWithRetry(ctx, logE, &DownloadParam{ Package: pkg, From 24e1d7e0f36616fac4a7f3303ac8651bdfa48c70 Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 07:44:59 +0900 Subject: [PATCH 55/73] refactor: refactor --- pkg/controller/vacuum/vacuum.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/controller/vacuum/vacuum.go b/pkg/controller/vacuum/vacuum.go index 7ff58e48b..6a79077d8 100644 --- a/pkg/controller/vacuum/vacuum.go +++ b/pkg/controller/vacuum/vacuum.go @@ -92,10 +92,7 @@ func (vc *Controller) StorePackage(logE *logrus.Entry, pkg *config.Package, pkgP logE.Warn("package is nil, skipping store package") return nil } - - vacuumPkg := vc.getVacuumPackage(pkg, pkgPath) - - return vc.handleAsyncStorePackage(logE, vacuumPkg) + return vc.handleAsyncStorePackage(logE, vc.getVacuumPackage(pkg, pkgPath)) } // getVacuumPackage converts a config @@ -142,9 +139,10 @@ func (vc *Controller) listExpiredPackages(ctx context.Context, logE *logrus.Entr return expired, nil } +const secondsInADay = 24 * 60 * 60 + // isPackageExpired checks if a package is expired based on the vacuum configuration. func (vc *Controller) isPackageExpired(pkg *PackageVacuumEntry) bool { - const secondsInADay = 24 * 60 * 60 threshold := vc.Param.VacuumDays * secondsInADay lastUsageTime := pkg.PackageEntry.LastUsageTime From a6353c1be8963bb7d63590f998fcd4361a9ef480 Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 09:45:30 +0900 Subject: [PATCH 56/73] fix: skip updating DB when a package install is skipped --- pkg/controller/exec/controller.go | 1 + pkg/controller/exec/exec.go | 7 +++++++ pkg/controller/which/which.go | 10 ++++++++++ pkg/controller/which/which_internal_test.go | 1 + pkg/controller/which/which_test.go | 3 +++ pkg/installpackage/download.go | 3 +++ pkg/installpackage/installer.go | 13 +++++-------- 7 files changed, 30 insertions(+), 8 deletions(-) diff --git a/pkg/controller/exec/controller.go b/pkg/controller/exec/controller.go index 6a6813e67..d1ad65543 100644 --- a/pkg/controller/exec/controller.go +++ b/pkg/controller/exec/controller.go @@ -64,4 +64,5 @@ type WhichController interface { type VacuumController interface { Close(logE *logrus.Entry) error + StorePackage(logE *logrus.Entry, pkg *config.Package, pkgPath string) error } diff --git a/pkg/controller/exec/exec.go b/pkg/controller/exec/exec.go index 2c50e206b..478c88c4b 100644 --- a/pkg/controller/exec/exec.go +++ b/pkg/controller/exec/exec.go @@ -99,6 +99,13 @@ func (c *Controller) install(ctx context.Context, logE *logrus.Entry, findResult }); err != nil { return fmt.Errorf("install the package: %w", err) } + if err := c.vacuum.StorePackage(logE, findResult.Package, findResult.PkgPath); err != nil { + logerr.WithError(logE, err).Error("store the package") + } + return c.checkExePath(ctx, logE, findResult) +} + +func (c *Controller) checkExePath(ctx context.Context, logE *logrus.Entry, findResult *which.FindResult) error { for i := range 10 { logE.Debug("check if exec file exists") if fi, err := c.fs.Stat(findResult.ExePath); err == nil { diff --git a/pkg/controller/which/which.go b/pkg/controller/which/which.go index a97b58711..8385714f0 100644 --- a/pkg/controller/which/which.go +++ b/pkg/controller/which/which.go @@ -19,6 +19,7 @@ type FindResult struct { File *registry.File Config *aqua.Config ExePath string + PkgPath string ConfigFilePath string EnableChecksum bool } @@ -161,12 +162,21 @@ func (c *Controller) findExecFileFromPkg(logE *logrus.Entry, registries map[stri return nil, nil //nolint:nilnil } + cPkg := &config.Package{ + Package: pkg, + PackageInfo: pkgInfo, + } + pkgPath, err := cPkg.PkgPath(c.runtime) + if err != nil { + return nil, fmt.Errorf("get a package path: %w", err) + } for _, file := range pkgInfo.GetFiles() { findResult, err := c.findExecFileFromFile(logE, exeName, pkg, pkgInfo, file) if err != nil { return nil, err } if findResult != nil { + findResult.PkgPath = pkgPath return findResult, nil } } diff --git a/pkg/controller/which/which_internal_test.go b/pkg/controller/which/which_internal_test.go index 81af78c27..48dc31a0e 100644 --- a/pkg/controller/which/which_internal_test.go +++ b/pkg/controller/which/which_internal_test.go @@ -51,6 +51,7 @@ func TestController_findExecFileFromPkg(t *testing.T) { //nolint:funlen Name: "kubectl", }, ExePath: filepath.Join("/home", "foo", ".local", "share", "aquaproj-aqua", "pkgs", "http", "storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl/kubectl"), + PkgPath: filepath.FromSlash("pkgs/http/storage.googleapis.com/kubernetes-release/release/v1.21.0/bin/linux/amd64/kubectl"), }, registries: map[string]*registry.Config{ "standard": { diff --git a/pkg/controller/which/which_test.go b/pkg/controller/which/which_test.go index 424adff5a..c599c4a5c 100644 --- a/pkg/controller/which/which_test.go +++ b/pkg/controller/which/which_test.go @@ -3,6 +3,7 @@ package which_test import ( "context" "net/http" + "path/filepath" "testing" "github.com/aquaproj/aqua/v2/pkg/config" @@ -105,6 +106,7 @@ packages: }, }, + PkgPath: filepath.FromSlash("pkgs/github_content/github.com/aquaproj/aqua-installer/v1.0.0/aqua-installer"), ExePath: "/home/foo/.local/share/aquaproj-aqua/pkgs/github_content/github.com/aquaproj/aqua-installer/v1.0.0/aqua-installer/aqua-installer", ConfigFilePath: "/home/foo/workspace/aqua.yaml", }, @@ -231,6 +233,7 @@ packages: }, }, }, + PkgPath: filepath.FromSlash("pkgs/github_content/github.com/aquaproj/aqua-installer/v1.0.0/aqua-installer"), ExePath: "/home/foo/.local/share/aquaproj-aqua/pkgs/github_content/github.com/aquaproj/aqua-installer/v1.0.0/aqua-installer/aqua-installer", ConfigFilePath: "/etc/aqua/aqua.yaml", }, diff --git a/pkg/installpackage/download.go b/pkg/installpackage/download.go index 148cb7cb8..f30183968 100644 --- a/pkg/installpackage/download.go +++ b/pkg/installpackage/download.go @@ -37,6 +37,9 @@ func (is *Installer) downloadWithRetry(ctx context.Context, logE *logrus.Entry, } return err } + if err := is.vacuum.StorePackage(logE, param.Package, param.PkgPath); err != nil { + logerr.WithError(logE, err).Error("store the package") + } return nil } if !finfo.IsDir() { diff --git a/pkg/installpackage/installer.go b/pkg/installpackage/installer.go index 2dfc06fb2..ccf521835 100644 --- a/pkg/installpackage/installer.go +++ b/pkg/installpackage/installer.go @@ -186,6 +186,7 @@ type DownloadParam struct { Checksum *checksum.Checksum Dest string Asset string + PkgPath string RequireChecksum bool } @@ -283,15 +284,11 @@ func (is *Installer) InstallPackage(ctx context.Context, logE *logrus.Entry, par if err != nil { return fmt.Errorf("get the package install path: %w", err) } - - if err := is.vacuum.StorePackage(logE, pkg, pkgPath); err != nil { - logerr.WithError(logE, err).Error("store the package") - } - pkgPath = filepath.Join(is.rootDir, pkgPath) - + absPkgPath := filepath.Join(is.rootDir, pkgPath) if err := is.downloadWithRetry(ctx, logE, &DownloadParam{ Package: pkg, - Dest: pkgPath, + Dest: absPkgPath, + PkgPath: pkgPath, Asset: assetName, Checksums: param.Checksums, RequireChecksum: param.RequireChecksum, @@ -303,5 +300,5 @@ func (is *Installer) InstallPackage(ctx context.Context, logE *logrus.Entry, par return err } - return is.checkFilesWrap(ctx, logE, param, pkgPath) + return is.checkFilesWrap(ctx, logE, param, absPkgPath) } From 1a3f886b1a59d9bd6dc88522fad25954b496c2b9 Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 10:59:18 +0900 Subject: [PATCH 57/73] refactor: separate code about DB --- pkg/controller/vacuum/controller.go | 151 ++----------------- pkg/controller/vacuum/queue_store.go | 15 +- pkg/controller/vacuum/vacuum.go | 54 +------ pkg/controller/vacuum/vacuum_db.go | 217 +++++++++++++++++++++++++++ 4 files changed, 245 insertions(+), 192 deletions(-) create mode 100644 pkg/controller/vacuum/vacuum_db.go diff --git a/pkg/controller/vacuum/controller.go b/pkg/controller/vacuum/controller.go index 61d341206..7453970dd 100644 --- a/pkg/controller/vacuum/controller.go +++ b/pkg/controller/vacuum/controller.go @@ -2,29 +2,24 @@ package vacuum import ( "context" - "fmt" "io" "os" - "path/filepath" "sync" "sync/atomic" - "time" "github.com/aquaproj/aqua/v2/pkg/config" - "github.com/aquaproj/aqua/v2/pkg/timer" "github.com/sirupsen/logrus" "github.com/spf13/afero" - "github.com/suzuki-shunsuke/logrus-error/logerr" bolt "go.etcd.io/bbolt" ) type Controller struct { - stdout io.Writer - dbMutex sync.RWMutex - db atomic.Pointer[bolt.DB] - Param *config.Param - fs afero.Fs - storeQueue *StoreQueue + stdout io.Writer + dbMutex sync.RWMutex + db atomic.Pointer[bolt.DB] + Param *config.Param + fs afero.Fs + d *DB } // New initializes a Controller with the given context, parameters, and dependencies. @@ -33,143 +28,23 @@ func New(ctx context.Context, param *config.Param, fs afero.Fs) *Controller { stdout: os.Stdout, Param: param, fs: fs, + d: NewDB(ctx, param, fs), } - vc.storeQueue = newStoreQueue(ctx, vc) return vc } -// getDB retrieves the database instance, initializing it if necessary. -func (vc *Controller) getDB() (*bolt.DB, error) { - if db := vc.db.Load(); db != nil { - return db, nil - } - - vc.dbMutex.Lock() - defer vc.dbMutex.Unlock() - - if db := vc.db.Load(); db != nil { - return db, nil - } - - const dbFileMode = 0o600 - db, err := bolt.Open(filepath.Join(vc.Param.RootDir, dbFile), dbFileMode, &bolt.Options{ - Timeout: 1 * time.Second, - }) - if err != nil { - return nil, fmt.Errorf("open database %v: %w", dbFile, err) - } - - if err := db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucketIfNotExists([]byte(bucketNamePkgs)) - if err != nil { - return fmt.Errorf("create bucket '%v' in '%v': %w", bucketNamePkgs, dbFile, err) - } - return nil - }); err != nil { - db.Close() - return nil, fmt.Errorf("create bucket: %w", err) - } - - vc.db.Store(db) - return db, nil -} - -// withDBRetry retries a database operation with exponential backoff. -func (vc *Controller) withDBRetry(ctx context.Context, logE *logrus.Entry, fn func(*bolt.Tx) error, dbAccessType DBAccessType) error { - const ( - retries = 2 - initialBackoff = 100 * time.Millisecond - exponentialBackoff = 2 - ) - backoff := initialBackoff - for i := range retries { - err := vc.withDB(logE, fn, dbAccessType) - if err == nil { - return nil - } - - logerr.WithError(logE, err).WithField("attempt", i+1).Warn("retrying database operation") - - if err := timer.Wait(ctx, backoff); err != nil { - return fmt.Errorf("wait for retrying database operation: %w", err) - } - backoff *= exponentialBackoff - } - - return fmt.Errorf("database operation failed after %d retries", retries) -} - -// withDB executes a function within a database transaction. -func (vc *Controller) withDB(logE *logrus.Entry, fn func(*bolt.Tx) error, dbAccessType DBAccessType) error { - db, err := vc.getDB() - if err != nil { - return err - } - if db == nil { - return nil - } - defer func() { - if err := vc.closeDB(); err != nil { - logerr.WithError(logE, err).Error("close database") - } - }() - - if dbAccessType == Update { - if err := db.Update(fn); err != nil { - return fmt.Errorf("update database: %w", err) - } - return nil - } - if err := db.View(fn); err != nil { - return fmt.Errorf("view database: %w", err) - } - return nil -} - -// Keep_DBOpen opens the database instance. This is used for testing purposes. -func (vc *Controller) TestKeepDBOpen() error { - const dbFileMode = 0o600 - if _, err := bolt.Open(filepath.Join(vc.Param.RootDir, dbFile), dbFileMode, &bolt.Options{ - Timeout: 1 * time.Second, - }); err != nil { - return fmt.Errorf("open database %v: %w", dbFile, err) - } - return nil -} - -// closeDB closes the database instance. -func (vc *Controller) closeDB() error { - vc.dbMutex.Lock() - defer vc.dbMutex.Unlock() - - if vc.db.Load() != nil { - if err := vc.db.Load().Close(); err != nil { - return fmt.Errorf("close database: %w", err) - } - vc.db.Store(nil) - } - - return nil -} - // Close closes the dependencies of the Controller. func (vc *Controller) Close(logE *logrus.Entry) error { if !vc.IsVacuumEnabled(logE) { return nil } logE.Debug("closing vacuum controller") - if vc.storeQueue != nil { - vc.storeQueue.close() + if vc.d.storeQueue != nil { + vc.d.storeQueue.close() } + return vc.d.Close() +} - vc.dbMutex.Lock() - defer vc.dbMutex.Unlock() - - if vc.db.Load() != nil { - if err := vc.db.Load().Close(); err != nil { - return fmt.Errorf("close database: %w", err) - } - vc.db.Store(nil) - } - return nil +func (vc *Controller) TestKeepDBOpen() error { + return vc.d.TestKeepDBOpen() } diff --git a/pkg/controller/vacuum/queue_store.go b/pkg/controller/vacuum/queue_store.go index f12a842c7..dd11dd362 100644 --- a/pkg/controller/vacuum/queue_store.go +++ b/pkg/controller/vacuum/queue_store.go @@ -3,6 +3,7 @@ package vacuum import ( "context" "sync" + "time" "github.com/sirupsen/logrus" "github.com/suzuki-shunsuke/logrus-error/logerr" @@ -18,13 +19,17 @@ type StoreRequest struct { type StoreQueue struct { taskQueue chan StoreRequest wg sync.WaitGroup - vc *Controller + vc Storage done chan struct{} closeOnce sync.Once } +type Storage interface { + StorePackageInternal(ctx context.Context, logE *logrus.Entry, pkg *Package, dateTime ...time.Time) error +} + // newStoreQueue initializes the task queue with a single worker. -func newStoreQueue(ctx context.Context, vc *Controller) *StoreQueue { +func newStoreQueue(ctx context.Context, vc Storage) *StoreQueue { const maxTasks = 100 sq := &StoreQueue{ taskQueue: make(chan StoreRequest, maxTasks), @@ -44,8 +49,7 @@ func (sq *StoreQueue) worker(ctx context.Context) { if !ok { return } - err := sq.vc.storePackageInternal(ctx, task.logE, task.pkg) - if err != nil { + if err := sq.vc.StorePackageInternal(ctx, task.logE, task.pkg); err != nil { logerr.WithError(task.logE, err).Error("store package asynchronously") } sq.wg.Done() @@ -53,8 +57,7 @@ func (sq *StoreQueue) worker(ctx context.Context) { // Process remaining tasks for len(sq.taskQueue) > 0 { task := <-sq.taskQueue - err := sq.vc.storePackageInternal(ctx, task.logE, task.pkg) - if err != nil { + if err := sq.vc.StorePackageInternal(ctx, task.logE, task.pkg); err != nil { logerr.WithError(task.logE, err).Error("store package asynchronously during shutdown") } sq.wg.Done() diff --git a/pkg/controller/vacuum/vacuum.go b/pkg/controller/vacuum/vacuum.go index 6a79077d8..2aa366f65 100644 --- a/pkg/controller/vacuum/vacuum.go +++ b/pkg/controller/vacuum/vacuum.go @@ -110,7 +110,7 @@ func (vc *Controller) handleAsyncStorePackage(logE *logrus.Entry, vacuumPkg *Pac if vacuumPkg == nil { return errors.New("vacuumPkg is nil") } - vc.storeQueue.enqueue(logE, vacuumPkg) + vc.d.storeQueue.enqueue(logE, vacuumPkg) return nil } @@ -156,7 +156,7 @@ func (vc *Controller) isPackageExpired(pkg *PackageVacuumEntry) bool { // listPackages lists all stored package entries. func (vc *Controller) listPackages(ctx context.Context, logE *logrus.Entry) ([]*PackageVacuumEntry, error) { - db, err := vc.getDB() + db, err := vc.d.getDB() if err != nil { return nil, err } @@ -167,7 +167,7 @@ func (vc *Controller) listPackages(ctx context.Context, logE *logrus.Entry) ([]* var pkgs []*PackageVacuumEntry - err = vc.withDBRetry(ctx, logE, func(tx *bbolt.Tx) error { + err = vc.d.withDBRetry(ctx, logE, func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(bucketNamePkgs)) if b == nil { return nil @@ -363,51 +363,9 @@ func (vc *Controller) processExpiredPackages(logE *logrus.Entry, expired []*Pack return pathsToRemove, errors } -// storePackageInternal stores package entries in the database. -func (vc *Controller) storePackageInternal(ctx context.Context, logE *logrus.Entry, pkg *Package, dateTime ...time.Time) error { - lastUsedTime := time.Now() - if len(dateTime) > 0 { - lastUsedTime = dateTime[0] - } - return vc.withDBRetry(ctx, logE, func(tx *bbolt.Tx) error { - b := tx.Bucket([]byte(bucketNamePkgs)) - if b == nil { - return errors.New("bucket not found") - } - logE.WithFields(logrus.Fields{ - "package_name": pkg.Name, - "package_version": pkg.Version, - "package_path": pkg.PkgPath, - }).Debug("storing package in vacuum database") - - pkgKey := pkg.PkgPath - pkgEntry := &PackageEntry{ - LastUsageTime: lastUsedTime, - Package: pkg, - } - - data, err := encodePackageEntry(pkgEntry) - if err != nil { - logerr.WithError(logE, err).WithFields( - logrus.Fields{ - "package_name": pkg.Name, - "package_version": pkg.Version, - "package_path": pkg.PkgPath, - }).Error("encode package") - return fmt.Errorf("encode package %s: %w", pkg.Name, err) - } - - if err := b.Put([]byte(pkgKey), data); err != nil { - logerr.WithError(logE, err).WithField("package_path", pkgKey).Error("store package in vacuum database") - return fmt.Errorf("store package %s: %w", pkg.Name, err) - } - return nil - }, Update) -} - // removePackages removes package entries from the database. func (vc *Controller) removePackages(ctx context.Context, logE *logrus.Entry, pkgs []string) error { - return vc.withDBRetry(ctx, logE, func(tx *bbolt.Tx) error { + return vc.d.withDBRetry(ctx, logE, func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(bucketNamePkgs)) if b == nil { return errors.New("bucket not found") @@ -462,13 +420,13 @@ func (vc *Controller) GetPackageLastUsed(ctx context.Context, logE *logrus.Entry // SetTimeStampPackage permit define a Timestamp for a package Manually. for testing purposes. func (vc *Controller) SetTimestampPackage(ctx context.Context, logE *logrus.Entry, pkg *config.Package, pkgPath string, datetime time.Time) error { vacuumPkg := vc.getVacuumPackage(pkg, pkgPath) - return vc.storePackageInternal(ctx, logE, vacuumPkg, datetime) + return vc.d.StorePackageInternal(ctx, logE, vacuumPkg, datetime) } // retrievePackageEntry retrieves a package entry from the database by key. for testing purposes. func (vc *Controller) retrievePackageEntry(ctx context.Context, logE *logrus.Entry, key string) (*PackageEntry, error) { var pkgEntry *PackageEntry - err := vc.withDBRetry(ctx, logE, func(tx *bbolt.Tx) error { + err := vc.d.withDBRetry(ctx, logE, func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(bucketNamePkgs)) if b == nil { return nil diff --git a/pkg/controller/vacuum/vacuum_db.go b/pkg/controller/vacuum/vacuum_db.go new file mode 100644 index 000000000..67391dc43 --- /dev/null +++ b/pkg/controller/vacuum/vacuum_db.go @@ -0,0 +1,217 @@ +package vacuum + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" + "sync/atomic" + "time" + + "github.com/aquaproj/aqua/v2/pkg/config" + "github.com/aquaproj/aqua/v2/pkg/timer" + "github.com/sirupsen/logrus" + "github.com/spf13/afero" + "github.com/suzuki-shunsuke/logrus-error/logerr" + "go.etcd.io/bbolt" + bolt "go.etcd.io/bbolt" +) + +type DB struct { + stdout io.Writer + dbMutex sync.RWMutex + db atomic.Pointer[bolt.DB] + Param *config.Param + fs afero.Fs + storeQueue *StoreQueue +} + +// NewDB initializes a Controller with the given context, parameters, and dependencies. +func NewDB(ctx context.Context, param *config.Param, fs afero.Fs) *DB { + db := &DB{ + stdout: os.Stdout, + Param: param, + fs: fs, + } + db.storeQueue = newStoreQueue(ctx, db) + return db +} + +// Get retrieves a package entry from the database by key. for testing purposes. +func (d *DB) Get(ctx context.Context, logE *logrus.Entry, key string) (*PackageEntry, error) { + var pkgEntry *PackageEntry + err := d.withDBRetry(ctx, logE, func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(bucketNamePkgs)) + if b == nil { + return nil + } + value := b.Get([]byte(key)) + if value == nil { + return nil + } + + var err error + pkgEntry, err = decodePackageEntry(value) + return err + }, View) + return pkgEntry, err +} + +// withDBRetry retries a database operation with exponential backoff. +func (d *DB) withDBRetry(ctx context.Context, logE *logrus.Entry, fn func(*bolt.Tx) error, dbAccessType DBAccessType) error { + const ( + retries = 2 + initialBackoff = 100 * time.Millisecond + exponentialBackoff = 2 + ) + backoff := initialBackoff + for i := range retries { + err := d.withDB(logE, fn, dbAccessType) + if err == nil { + return nil + } + + logerr.WithError(logE, err).WithField("attempt", i+1).Warn("retrying database operation") + + if err := timer.Wait(ctx, backoff); err != nil { + return fmt.Errorf("wait for retrying database operation: %w", err) + } + backoff *= exponentialBackoff + } + + return fmt.Errorf("database operation failed after %d retries", retries) +} + +// withDB executes a function within a database transaction. +func (d *DB) withDB(logE *logrus.Entry, fn func(*bolt.Tx) error, dbAccessType DBAccessType) error { + db, err := d.getDB() + if err != nil { + return err + } + if db == nil { + return nil + } + defer func() { + if err := d.Close(); err != nil { + logerr.WithError(logE, err).Error("close database") + } + }() + + if dbAccessType == Update { + if err := db.Update(fn); err != nil { + return fmt.Errorf("update database: %w", err) + } + return nil + } + if err := db.View(fn); err != nil { + return fmt.Errorf("view database: %w", err) + } + return nil +} + +// getDB retrieves the database instance, initializing it if necessary. +func (d *DB) getDB() (*bolt.DB, error) { + if db := d.db.Load(); db != nil { + return db, nil + } + + d.dbMutex.Lock() + defer d.dbMutex.Unlock() + + if db := d.db.Load(); db != nil { + return db, nil + } + + const dbFileMode = 0o600 + db, err := bolt.Open(filepath.Join(d.Param.RootDir, dbFile), dbFileMode, &bolt.Options{ + Timeout: 1 * time.Second, + }) + if err != nil { + return nil, fmt.Errorf("open database %v: %w", dbFile, err) + } + + if err := db.Update(func(tx *bolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte(bucketNamePkgs)) + if err != nil { + return fmt.Errorf("create bucket '%v' in '%v': %w", bucketNamePkgs, dbFile, err) + } + return nil + }); err != nil { + db.Close() + return nil, fmt.Errorf("create bucket: %w", err) + } + + d.db.Store(db) + return db, nil +} + +// Keep_DBOpen opens the database instance. This is used for testing purposes. +func (d *DB) TestKeepDBOpen() error { + const dbFileMode = 0o600 + if _, err := bolt.Open(filepath.Join(d.Param.RootDir, dbFile), dbFileMode, &bolt.Options{ + Timeout: 1 * time.Second, + }); err != nil { + return fmt.Errorf("open database %v: %w", dbFile, err) + } + return nil +} + +// Close closes the database instance. +func (d *DB) Close() error { + d.dbMutex.Lock() + defer d.dbMutex.Unlock() + + if d.db.Load() != nil { + if err := d.db.Load().Close(); err != nil { + return fmt.Errorf("close database: %w", err) + } + d.db.Store(nil) + } + + return nil +} + +// StorePackageInternal stores package entries in the database. +func (d *DB) StorePackageInternal(ctx context.Context, logE *logrus.Entry, pkg *Package, dateTime ...time.Time) error { + lastUsedTime := time.Now() + if len(dateTime) > 0 { + lastUsedTime = dateTime[0] + } + return d.withDBRetry(ctx, logE, func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(bucketNamePkgs)) + if b == nil { + return errors.New("bucket not found") + } + logE.WithFields(logrus.Fields{ + "package_name": pkg.Name, + "package_version": pkg.Version, + "package_path": pkg.PkgPath, + }).Debug("storing package in vacuum database") + + pkgKey := pkg.PkgPath + pkgEntry := &PackageEntry{ + LastUsageTime: lastUsedTime, + Package: pkg, + } + + data, err := encodePackageEntry(pkgEntry) + if err != nil { + logerr.WithError(logE, err).WithFields( + logrus.Fields{ + "package_name": pkg.Name, + "package_version": pkg.Version, + "package_path": pkg.PkgPath, + }).Error("encode package") + return fmt.Errorf("encode package %s: %w", pkg.Name, err) + } + + if err := b.Put([]byte(pkgKey), data); err != nil { + logerr.WithError(logE, err).WithField("package_path", pkgKey).Error("store package in vacuum database") + return fmt.Errorf("store package %s: %w", pkg.Name, err) + } + return nil + }, Update) +} From ffbd7e0c2c3938dfca85c2c12bb3bd96794a9062 Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 11:01:05 +0900 Subject: [PATCH 58/73] refactor: rename a method --- pkg/controller/vacuum/queue_store.go | 6 +++--- pkg/controller/vacuum/vacuum.go | 3 +-- pkg/controller/vacuum/vacuum_db.go | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/controller/vacuum/queue_store.go b/pkg/controller/vacuum/queue_store.go index dd11dd362..5dacc01f7 100644 --- a/pkg/controller/vacuum/queue_store.go +++ b/pkg/controller/vacuum/queue_store.go @@ -25,7 +25,7 @@ type StoreQueue struct { } type Storage interface { - StorePackageInternal(ctx context.Context, logE *logrus.Entry, pkg *Package, dateTime ...time.Time) error + Store(ctx context.Context, logE *logrus.Entry, pkg *Package, dateTime ...time.Time) error } // newStoreQueue initializes the task queue with a single worker. @@ -49,7 +49,7 @@ func (sq *StoreQueue) worker(ctx context.Context) { if !ok { return } - if err := sq.vc.StorePackageInternal(ctx, task.logE, task.pkg); err != nil { + if err := sq.vc.Store(ctx, task.logE, task.pkg); err != nil { logerr.WithError(task.logE, err).Error("store package asynchronously") } sq.wg.Done() @@ -57,7 +57,7 @@ func (sq *StoreQueue) worker(ctx context.Context) { // Process remaining tasks for len(sq.taskQueue) > 0 { task := <-sq.taskQueue - if err := sq.vc.StorePackageInternal(ctx, task.logE, task.pkg); err != nil { + if err := sq.vc.Store(ctx, task.logE, task.pkg); err != nil { logerr.WithError(task.logE, err).Error("store package asynchronously during shutdown") } sq.wg.Done() diff --git a/pkg/controller/vacuum/vacuum.go b/pkg/controller/vacuum/vacuum.go index 2aa366f65..e08624b6a 100644 --- a/pkg/controller/vacuum/vacuum.go +++ b/pkg/controller/vacuum/vacuum.go @@ -419,8 +419,7 @@ func (vc *Controller) GetPackageLastUsed(ctx context.Context, logE *logrus.Entry // SetTimeStampPackage permit define a Timestamp for a package Manually. for testing purposes. func (vc *Controller) SetTimestampPackage(ctx context.Context, logE *logrus.Entry, pkg *config.Package, pkgPath string, datetime time.Time) error { - vacuumPkg := vc.getVacuumPackage(pkg, pkgPath) - return vc.d.StorePackageInternal(ctx, logE, vacuumPkg, datetime) + return vc.d.Store(ctx, logE, vc.getVacuumPackage(pkg, pkgPath), datetime) } // retrievePackageEntry retrieves a package entry from the database by key. for testing purposes. diff --git a/pkg/controller/vacuum/vacuum_db.go b/pkg/controller/vacuum/vacuum_db.go index 67391dc43..eb5497e62 100644 --- a/pkg/controller/vacuum/vacuum_db.go +++ b/pkg/controller/vacuum/vacuum_db.go @@ -175,7 +175,7 @@ func (d *DB) Close() error { } // StorePackageInternal stores package entries in the database. -func (d *DB) StorePackageInternal(ctx context.Context, logE *logrus.Entry, pkg *Package, dateTime ...time.Time) error { +func (d *DB) Store(ctx context.Context, logE *logrus.Entry, pkg *Package, dateTime ...time.Time) error { lastUsedTime := time.Now() if len(dateTime) > 0 { lastUsedTime = dateTime[0] From 6e650726f9c79219ab10f630c7c10499b20d423b Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 11:13:00 +0900 Subject: [PATCH 59/73] refactor: separate code about DB --- pkg/controller/vacuum/vacuum.go | 27 +++++++++------------------ pkg/controller/vacuum/vacuum_db.go | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/pkg/controller/vacuum/vacuum.go b/pkg/controller/vacuum/vacuum.go index e08624b6a..b66e71a83 100644 --- a/pkg/controller/vacuum/vacuum.go +++ b/pkg/controller/vacuum/vacuum.go @@ -17,15 +17,6 @@ import ( "go.etcd.io/bbolt" ) -type DBAccessType string - -const ( - dbFile string = "vacuum.db" - bucketNamePkgs string = "packages" - View DBAccessType = "view" - Update DBAccessType = "update" -) - type PackageVacuumEntries []PackageVacuumEntry type PackageVacuumEntry struct { @@ -167,8 +158,8 @@ func (vc *Controller) listPackages(ctx context.Context, logE *logrus.Entry) ([]* var pkgs []*PackageVacuumEntry - err = vc.d.withDBRetry(ctx, logE, func(tx *bbolt.Tx) error { - b := tx.Bucket([]byte(bucketNamePkgs)) + err = vc.d.view(ctx, logE, func(tx *bbolt.Tx) error { + b := vc.d.Bucket(tx) if b == nil { return nil } @@ -184,7 +175,7 @@ func (vc *Controller) listPackages(ctx context.Context, logE *logrus.Entry) ([]* }) return nil }) - }, View) + }) return pkgs, err } @@ -365,8 +356,8 @@ func (vc *Controller) processExpiredPackages(logE *logrus.Entry, expired []*Pack // removePackages removes package entries from the database. func (vc *Controller) removePackages(ctx context.Context, logE *logrus.Entry, pkgs []string) error { - return vc.d.withDBRetry(ctx, logE, func(tx *bbolt.Tx) error { - b := tx.Bucket([]byte(bucketNamePkgs)) + return vc.d.update(ctx, logE, func(tx *bbolt.Tx) error { + b := vc.d.Bucket(tx) if b == nil { return errors.New("bucket not found") } @@ -378,7 +369,7 @@ func (vc *Controller) removePackages(ctx context.Context, logE *logrus.Entry, pk logE.WithField("pkgKey", key).Info("removed package from vacuum database") } return nil - }, Update) + }) } // removePackageVersionPath removes the specified package version directory and its parent directory if it becomes empty. @@ -425,8 +416,8 @@ func (vc *Controller) SetTimestampPackage(ctx context.Context, logE *logrus.Entr // retrievePackageEntry retrieves a package entry from the database by key. for testing purposes. func (vc *Controller) retrievePackageEntry(ctx context.Context, logE *logrus.Entry, key string) (*PackageEntry, error) { var pkgEntry *PackageEntry - err := vc.d.withDBRetry(ctx, logE, func(tx *bbolt.Tx) error { - b := tx.Bucket([]byte(bucketNamePkgs)) + err := vc.d.view(ctx, logE, func(tx *bbolt.Tx) error { + b := vc.d.Bucket(tx) if b == nil { return nil } @@ -438,6 +429,6 @@ func (vc *Controller) retrievePackageEntry(ctx context.Context, logE *logrus.Ent var err error pkgEntry, err = decodePackageEntry(value) return err - }, View) + }) return pkgEntry, err } diff --git a/pkg/controller/vacuum/vacuum_db.go b/pkg/controller/vacuum/vacuum_db.go index eb5497e62..ec14c927d 100644 --- a/pkg/controller/vacuum/vacuum_db.go +++ b/pkg/controller/vacuum/vacuum_db.go @@ -20,6 +20,15 @@ import ( bolt "go.etcd.io/bbolt" ) +const ( + dbFile string = "vacuum.db" + bucketNamePkgs string = "packages" + View DBAccessType = "view" + Update DBAccessType = "update" +) + +type DBAccessType string + type DB struct { stdout io.Writer dbMutex sync.RWMutex @@ -40,6 +49,10 @@ func NewDB(ctx context.Context, param *config.Param, fs afero.Fs) *DB { return db } +func (d *DB) Bucket(tx *bbolt.Tx) *bbolt.Bucket { + return tx.Bucket([]byte(bucketNamePkgs)) +} + // Get retrieves a package entry from the database by key. for testing purposes. func (d *DB) Get(ctx context.Context, logE *logrus.Entry, key string) (*PackageEntry, error) { var pkgEntry *PackageEntry @@ -60,6 +73,14 @@ func (d *DB) Get(ctx context.Context, logE *logrus.Entry, key string) (*PackageE return pkgEntry, err } +func (d *DB) view(ctx context.Context, logE *logrus.Entry, fn func(*bolt.Tx) error) error { + return d.withDBRetry(ctx, logE, fn, View) +} + +func (d *DB) update(ctx context.Context, logE *logrus.Entry, fn func(*bolt.Tx) error) error { + return d.withDBRetry(ctx, logE, fn, Update) +} + // withDBRetry retries a database operation with exponential backoff. func (d *DB) withDBRetry(ctx context.Context, logE *logrus.Entry, fn func(*bolt.Tx) error, dbAccessType DBAccessType) error { const ( From 596a6a4cd5ad447d15d018219e56074fb3e1ad57 Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 11:14:50 +0900 Subject: [PATCH 60/73] fix: fix lint errors --- pkg/controller/vacuum/controller.go | 13 ++++--------- pkg/controller/vacuum/vacuum_db.go | 3 +-- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/pkg/controller/vacuum/controller.go b/pkg/controller/vacuum/controller.go index 7453970dd..ad08a50a8 100644 --- a/pkg/controller/vacuum/controller.go +++ b/pkg/controller/vacuum/controller.go @@ -4,22 +4,17 @@ import ( "context" "io" "os" - "sync" - "sync/atomic" "github.com/aquaproj/aqua/v2/pkg/config" "github.com/sirupsen/logrus" "github.com/spf13/afero" - bolt "go.etcd.io/bbolt" ) type Controller struct { - stdout io.Writer - dbMutex sync.RWMutex - db atomic.Pointer[bolt.DB] - Param *config.Param - fs afero.Fs - d *DB + stdout io.Writer + Param *config.Param + fs afero.Fs + d *DB } // New initializes a Controller with the given context, parameters, and dependencies. diff --git a/pkg/controller/vacuum/vacuum_db.go b/pkg/controller/vacuum/vacuum_db.go index ec14c927d..04272f02a 100644 --- a/pkg/controller/vacuum/vacuum_db.go +++ b/pkg/controller/vacuum/vacuum_db.go @@ -17,7 +17,6 @@ import ( "github.com/spf13/afero" "github.com/suzuki-shunsuke/logrus-error/logerr" "go.etcd.io/bbolt" - bolt "go.etcd.io/bbolt" ) const ( @@ -32,7 +31,7 @@ type DBAccessType string type DB struct { stdout io.Writer dbMutex sync.RWMutex - db atomic.Pointer[bolt.DB] + db atomic.Pointer[bbolt.DB] Param *config.Param fs afero.Fs storeQueue *StoreQueue From 3070a5dfb398fa3b3421458910f760ef3f57ca1e Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 11:16:37 +0900 Subject: [PATCH 61/73] fix: fix compile errors --- pkg/controller/vacuum/vacuum_db.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/controller/vacuum/vacuum_db.go b/pkg/controller/vacuum/vacuum_db.go index 04272f02a..5e1c0f74f 100644 --- a/pkg/controller/vacuum/vacuum_db.go +++ b/pkg/controller/vacuum/vacuum_db.go @@ -72,16 +72,16 @@ func (d *DB) Get(ctx context.Context, logE *logrus.Entry, key string) (*PackageE return pkgEntry, err } -func (d *DB) view(ctx context.Context, logE *logrus.Entry, fn func(*bolt.Tx) error) error { +func (d *DB) view(ctx context.Context, logE *logrus.Entry, fn func(*bbolt.Tx) error) error { return d.withDBRetry(ctx, logE, fn, View) } -func (d *DB) update(ctx context.Context, logE *logrus.Entry, fn func(*bolt.Tx) error) error { +func (d *DB) update(ctx context.Context, logE *logrus.Entry, fn func(*bbolt.Tx) error) error { return d.withDBRetry(ctx, logE, fn, Update) } // withDBRetry retries a database operation with exponential backoff. -func (d *DB) withDBRetry(ctx context.Context, logE *logrus.Entry, fn func(*bolt.Tx) error, dbAccessType DBAccessType) error { +func (d *DB) withDBRetry(ctx context.Context, logE *logrus.Entry, fn func(*bbolt.Tx) error, dbAccessType DBAccessType) error { const ( retries = 2 initialBackoff = 100 * time.Millisecond @@ -106,7 +106,7 @@ func (d *DB) withDBRetry(ctx context.Context, logE *logrus.Entry, fn func(*bolt. } // withDB executes a function within a database transaction. -func (d *DB) withDB(logE *logrus.Entry, fn func(*bolt.Tx) error, dbAccessType DBAccessType) error { +func (d *DB) withDB(logE *logrus.Entry, fn func(*bbolt.Tx) error, dbAccessType DBAccessType) error { db, err := d.getDB() if err != nil { return err @@ -133,7 +133,7 @@ func (d *DB) withDB(logE *logrus.Entry, fn func(*bolt.Tx) error, dbAccessType DB } // getDB retrieves the database instance, initializing it if necessary. -func (d *DB) getDB() (*bolt.DB, error) { +func (d *DB) getDB() (*bbolt.DB, error) { if db := d.db.Load(); db != nil { return db, nil } @@ -146,14 +146,14 @@ func (d *DB) getDB() (*bolt.DB, error) { } const dbFileMode = 0o600 - db, err := bolt.Open(filepath.Join(d.Param.RootDir, dbFile), dbFileMode, &bolt.Options{ + db, err := bbolt.Open(filepath.Join(d.Param.RootDir, dbFile), dbFileMode, &bbolt.Options{ Timeout: 1 * time.Second, }) if err != nil { return nil, fmt.Errorf("open database %v: %w", dbFile, err) } - if err := db.Update(func(tx *bolt.Tx) error { + if err := db.Update(func(tx *bbolt.Tx) error { _, err := tx.CreateBucketIfNotExists([]byte(bucketNamePkgs)) if err != nil { return fmt.Errorf("create bucket '%v' in '%v': %w", bucketNamePkgs, dbFile, err) @@ -171,7 +171,7 @@ func (d *DB) getDB() (*bolt.DB, error) { // Keep_DBOpen opens the database instance. This is used for testing purposes. func (d *DB) TestKeepDBOpen() error { const dbFileMode = 0o600 - if _, err := bolt.Open(filepath.Join(d.Param.RootDir, dbFile), dbFileMode, &bolt.Options{ + if _, err := bbolt.Open(filepath.Join(d.Param.RootDir, dbFile), dbFileMode, &bbolt.Options{ Timeout: 1 * time.Second, }); err != nil { return fmt.Errorf("open database %v: %w", dbFile, err) From 2fb9ef44104b69f1c4efa0d899da9abe836d82f3 Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 12:13:57 +0900 Subject: [PATCH 62/73] refactor: refactor --- pkg/controller/vacuum/vacuum_db.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/controller/vacuum/vacuum_db.go b/pkg/controller/vacuum/vacuum_db.go index 5e1c0f74f..f7a694418 100644 --- a/pkg/controller/vacuum/vacuum_db.go +++ b/pkg/controller/vacuum/vacuum_db.go @@ -194,13 +194,13 @@ func (d *DB) Close() error { return nil } -// StorePackageInternal stores package entries in the database. +// Store stores package entries in the database. func (d *DB) Store(ctx context.Context, logE *logrus.Entry, pkg *Package, dateTime ...time.Time) error { lastUsedTime := time.Now() if len(dateTime) > 0 { lastUsedTime = dateTime[0] } - return d.withDBRetry(ctx, logE, func(tx *bbolt.Tx) error { + return d.update(ctx, logE, func(tx *bbolt.Tx) error { b := tx.Bucket([]byte(bucketNamePkgs)) if b == nil { return errors.New("bucket not found") @@ -233,5 +233,5 @@ func (d *DB) Store(ctx context.Context, logE *logrus.Entry, pkg *Package, dateTi return fmt.Errorf("store package %s: %w", pkg.Name, err) } return nil - }, Update) + }) } From 3c24a2a093839c9b2b2e0e0df3496541b78c34c7 Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 13:49:17 +0900 Subject: [PATCH 63/73] refactor: refactor --- pkg/controller/vacuum/queue_store.go | 6 +- pkg/controller/vacuum/vacuum_db.go | 132 +++++++++++++-------------- 2 files changed, 67 insertions(+), 71 deletions(-) diff --git a/pkg/controller/vacuum/queue_store.go b/pkg/controller/vacuum/queue_store.go index 5dacc01f7..da1e6fb74 100644 --- a/pkg/controller/vacuum/queue_store.go +++ b/pkg/controller/vacuum/queue_store.go @@ -25,7 +25,7 @@ type StoreQueue struct { } type Storage interface { - Store(ctx context.Context, logE *logrus.Entry, pkg *Package, dateTime ...time.Time) error + Store(ctx context.Context, logE *logrus.Entry, pkg *Package, lastUsedTime time.Time) error } // newStoreQueue initializes the task queue with a single worker. @@ -49,7 +49,7 @@ func (sq *StoreQueue) worker(ctx context.Context) { if !ok { return } - if err := sq.vc.Store(ctx, task.logE, task.pkg); err != nil { + if err := sq.vc.Store(ctx, task.logE, task.pkg, time.Now()); err != nil { logerr.WithError(task.logE, err).Error("store package asynchronously") } sq.wg.Done() @@ -57,7 +57,7 @@ func (sq *StoreQueue) worker(ctx context.Context) { // Process remaining tasks for len(sq.taskQueue) > 0 { task := <-sq.taskQueue - if err := sq.vc.Store(ctx, task.logE, task.pkg); err != nil { + if err := sq.vc.Store(ctx, task.logE, task.pkg, time.Now()); err != nil { logerr.WithError(task.logE, err).Error("store package asynchronously during shutdown") } sq.wg.Done() diff --git a/pkg/controller/vacuum/vacuum_db.go b/pkg/controller/vacuum/vacuum_db.go index f7a694418..6725eb183 100644 --- a/pkg/controller/vacuum/vacuum_db.go +++ b/pkg/controller/vacuum/vacuum_db.go @@ -72,6 +72,70 @@ func (d *DB) Get(ctx context.Context, logE *logrus.Entry, key string) (*PackageE return pkgEntry, err } +// Store stores package entries in the database. +func (d *DB) Store(ctx context.Context, logE *logrus.Entry, pkg *Package, lastUsedTime time.Time) error { + return d.update(ctx, logE, func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(bucketNamePkgs)) + if b == nil { + return errors.New("bucket not found") + } + logE.WithFields(logrus.Fields{ + "package_name": pkg.Name, + "package_version": pkg.Version, + "package_path": pkg.PkgPath, + }).Debug("storing package in vacuum database") + + pkgKey := pkg.PkgPath + pkgEntry := &PackageEntry{ + LastUsageTime: lastUsedTime, + Package: pkg, + } + + data, err := encodePackageEntry(pkgEntry) + if err != nil { + logerr.WithError(logE, err).WithFields( + logrus.Fields{ + "package_name": pkg.Name, + "package_version": pkg.Version, + "package_path": pkg.PkgPath, + }).Error("encode package") + return fmt.Errorf("encode package %s: %w", pkg.Name, err) + } + + if err := b.Put([]byte(pkgKey), data); err != nil { + logerr.WithError(logE, err).WithField("package_path", pkgKey).Error("store package in vacuum database") + return fmt.Errorf("store package %s: %w", pkg.Name, err) + } + return nil + }) +} + +// Close closes the database instance. +func (d *DB) Close() error { + d.dbMutex.Lock() + defer d.dbMutex.Unlock() + + if d.db.Load() != nil { + if err := d.db.Load().Close(); err != nil { + return fmt.Errorf("close database: %w", err) + } + d.db.Store(nil) + } + + return nil +} + +// TesetKeepDBOpen opens the database instance. This is used for testing purposes. +func (d *DB) TestKeepDBOpen() error { + const dbFileMode = 0o600 + if _, err := bbolt.Open(filepath.Join(d.Param.RootDir, dbFile), dbFileMode, &bbolt.Options{ + Timeout: 1 * time.Second, + }); err != nil { + return fmt.Errorf("open database %v: %w", dbFile, err) + } + return nil +} + func (d *DB) view(ctx context.Context, logE *logrus.Entry, fn func(*bbolt.Tx) error) error { return d.withDBRetry(ctx, logE, fn, View) } @@ -167,71 +231,3 @@ func (d *DB) getDB() (*bbolt.DB, error) { d.db.Store(db) return db, nil } - -// Keep_DBOpen opens the database instance. This is used for testing purposes. -func (d *DB) TestKeepDBOpen() error { - const dbFileMode = 0o600 - if _, err := bbolt.Open(filepath.Join(d.Param.RootDir, dbFile), dbFileMode, &bbolt.Options{ - Timeout: 1 * time.Second, - }); err != nil { - return fmt.Errorf("open database %v: %w", dbFile, err) - } - return nil -} - -// Close closes the database instance. -func (d *DB) Close() error { - d.dbMutex.Lock() - defer d.dbMutex.Unlock() - - if d.db.Load() != nil { - if err := d.db.Load().Close(); err != nil { - return fmt.Errorf("close database: %w", err) - } - d.db.Store(nil) - } - - return nil -} - -// Store stores package entries in the database. -func (d *DB) Store(ctx context.Context, logE *logrus.Entry, pkg *Package, dateTime ...time.Time) error { - lastUsedTime := time.Now() - if len(dateTime) > 0 { - lastUsedTime = dateTime[0] - } - return d.update(ctx, logE, func(tx *bbolt.Tx) error { - b := tx.Bucket([]byte(bucketNamePkgs)) - if b == nil { - return errors.New("bucket not found") - } - logE.WithFields(logrus.Fields{ - "package_name": pkg.Name, - "package_version": pkg.Version, - "package_path": pkg.PkgPath, - }).Debug("storing package in vacuum database") - - pkgKey := pkg.PkgPath - pkgEntry := &PackageEntry{ - LastUsageTime: lastUsedTime, - Package: pkg, - } - - data, err := encodePackageEntry(pkgEntry) - if err != nil { - logerr.WithError(logE, err).WithFields( - logrus.Fields{ - "package_name": pkg.Name, - "package_version": pkg.Version, - "package_path": pkg.PkgPath, - }).Error("encode package") - return fmt.Errorf("encode package %s: %w", pkg.Name, err) - } - - if err := b.Put([]byte(pkgKey), data); err != nil { - logerr.WithError(logE, err).WithField("package_path", pkgKey).Error("store package in vacuum database") - return fmt.Errorf("store package %s: %w", pkg.Name, err) - } - return nil - }) -} From c48052c7558f75f3155727ed0934ae3dbdbe44d1 Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 14:20:59 +0900 Subject: [PATCH 64/73] refactor: close database in cli packages --- pkg/cli/exec/command.go | 7 ++- pkg/cli/install/command.go | 5 +- pkg/cli/vacuum/command.go | 15 ++++- pkg/controller/exec/exec.go | 7 ++- pkg/controller/install/install.go | 7 ++- pkg/controller/vacuum/controller.go | 17 ------ pkg/controller/vacuum/vacuum.go | 92 +++++++++++++++++------------ pkg/controller/vacuum/vacuum_db.go | 4 +- 8 files changed, 87 insertions(+), 67 deletions(-) diff --git a/pkg/cli/exec/command.go b/pkg/cli/exec/command.go index 16f136306..b73d875e8 100644 --- a/pkg/cli/exec/command.go +++ b/pkg/cli/exec/command.go @@ -42,8 +42,10 @@ func (i *command) action(c *cli.Context) error { } defer profiler.Stop() + logE := i.r.LogE + param := &config.Param{} - if err := util.SetParam(c, i.r.LogE, "exec", param, i.r.LDFlags); err != nil { + if err := util.SetParam(c, logE, "exec", param, i.r.LDFlags); err != nil { return fmt.Errorf("parse the command line arguments: %w", err) } ctrl, err := controller.InitializeExecCommandController(c.Context, param, http.DefaultClient, i.r.Runtime) @@ -54,5 +56,6 @@ func (i *command) action(c *cli.Context) error { if err != nil { return fmt.Errorf("parse args: %w", err) } - return ctrl.Exec(c.Context, i.r.LogE, param, exeName, args...) //nolint:wrapcheck + defer ctrl.CloseVacuum(logE) + return ctrl.Exec(c.Context, logE, param, exeName, args...) //nolint:wrapcheck } diff --git a/pkg/cli/install/command.go b/pkg/cli/install/command.go index fb055b739..b472df7e1 100644 --- a/pkg/cli/install/command.go +++ b/pkg/cli/install/command.go @@ -80,13 +80,16 @@ func (i *command) action(c *cli.Context) error { } defer profiler.Stop() + logE := i.r.LogE + param := &config.Param{} - if err := util.SetParam(c, i.r.LogE, "install", param, i.r.LDFlags); err != nil { + if err := util.SetParam(c, logE, "install", param, i.r.LDFlags); err != nil { return fmt.Errorf("parse the command line arguments: %w", err) } ctrl, err := controller.InitializeInstallCommandController(c.Context, param, http.DefaultClient, i.r.Runtime) if err != nil { return fmt.Errorf("initialize a InstallController: %w", err) } + defer ctrl.CloseVacuum(logE) return ctrl.Install(c.Context, i.r.LogE, param) //nolint:wrapcheck } diff --git a/pkg/cli/vacuum/command.go b/pkg/cli/vacuum/command.go index 1ed9ea348..25c34ea00 100644 --- a/pkg/cli/vacuum/command.go +++ b/pkg/cli/vacuum/command.go @@ -9,6 +9,7 @@ import ( "github.com/aquaproj/aqua/v2/pkg/cli/util" "github.com/aquaproj/aqua/v2/pkg/config" "github.com/aquaproj/aqua/v2/pkg/controller" + "github.com/suzuki-shunsuke/logrus-error/logerr" "github.com/urfave/cli/v2" ) @@ -74,8 +75,10 @@ func (i *command) action(c *cli.Context) error { } defer profiler.Stop() + logE := i.r.LogE + param := &config.Param{} - if err := util.SetParam(c, i.r.LogE, "vacuum", param, i.r.LDFlags); err != nil { + if err := util.SetParam(c, logE, "vacuum", param, i.r.LDFlags); err != nil { return fmt.Errorf("parse the command line arguments: %w", err) } @@ -85,15 +88,21 @@ func (i *command) action(c *cli.Context) error { ctrl := controller.InitializeVacuumCommandController(c.Context, param, http.DefaultClient, i.r.Runtime) + defer func() { + if err := ctrl.Close(logE); err != nil { + logerr.WithError(logE, err).Error("close vacuum controller") + } + }() + if c.Command.Name == "show" { - if err := ctrl.ListPackages(c.Context, i.r.LogE, c.Bool("expired")); err != nil { + if err := ctrl.ListPackages(c.Context, logE, c.Bool("expired")); err != nil { return fmt.Errorf("show packages: %w", err) } return nil } if c.Command.Name == "run" { - if err := ctrl.Vacuum(c.Context, i.r.LogE); err != nil { + if err := ctrl.Vacuum(c.Context, logE); err != nil { return fmt.Errorf("run: %w", err) } } diff --git a/pkg/controller/exec/exec.go b/pkg/controller/exec/exec.go index 478c88c4b..61ee7bf61 100644 --- a/pkg/controller/exec/exec.go +++ b/pkg/controller/exec/exec.go @@ -44,6 +44,10 @@ func (c *Controller) Exec(ctx context.Context, logE *logrus.Entry, param *config return c.execCommandWithRetry(ctx, logE, findResult.ExePath, args...) } + if err := c.vacuum.StorePackage(logE, findResult.Package, findResult.PkgPath); err != nil { + logerr.WithError(logE, err).Error("store the package") + } + logE = logE.WithFields(logrus.Fields{ "package_name": findResult.Package.Package.Name, "package_version": findResult.Package.Package.Version, @@ -62,11 +66,10 @@ func (c *Controller) Exec(ctx context.Context, logE *logrus.Entry, param *config if err := c.install(ctx, logE, findResult, policyCfgs, param); err != nil { return err } - c.vacuumClose(logE) return c.execCommandWithRetry(ctx, logE, findResult.ExePath, args...) } -func (c *Controller) vacuumClose(logE *logrus.Entry) { +func (c *Controller) CloseVacuum(logE *logrus.Entry) { if err := c.vacuum.Close(logE); err != nil { logerr.WithError(logE, err).Error("close the vacuum") } diff --git a/pkg/controller/install/install.go b/pkg/controller/install/install.go index adc68e807..7eed3ab7e 100644 --- a/pkg/controller/install/install.go +++ b/pkg/controller/install/install.go @@ -53,6 +53,12 @@ func (c *Controller) Install(ctx context.Context, logE *logrus.Entry, param *con return c.installAll(ctx, logE, param, policyCfgs, globalPolicyPaths) } +func (c *Controller) CloseVacuum(logE *logrus.Entry) { + if err := c.vacuum.Close(logE); err != nil { + logE.WithError(err).Error("close the vacuum") + } +} + func (c *Controller) mkBinDir() error { if err := osfile.MkdirAll(c.fs, filepath.Join(c.rootDir, "bin")); err != nil { return fmt.Errorf("create the directory: %w", err) @@ -85,7 +91,6 @@ func (c *Controller) installAll(ctx context.Context, logE *logrus.Entry, param * } func (c *Controller) install(ctx context.Context, logE *logrus.Entry, cfgFilePath string, policyConfigs []*policy.Config, param *config.Param) error { - defer c.vacuum.Close(logE) cfg := &aqua.Config{} if cfgFilePath == "" { return finder.ErrConfigFileNotFound diff --git a/pkg/controller/vacuum/controller.go b/pkg/controller/vacuum/controller.go index ad08a50a8..8f705923a 100644 --- a/pkg/controller/vacuum/controller.go +++ b/pkg/controller/vacuum/controller.go @@ -6,7 +6,6 @@ import ( "os" "github.com/aquaproj/aqua/v2/pkg/config" - "github.com/sirupsen/logrus" "github.com/spf13/afero" ) @@ -27,19 +26,3 @@ func New(ctx context.Context, param *config.Param, fs afero.Fs) *Controller { } return vc } - -// Close closes the dependencies of the Controller. -func (vc *Controller) Close(logE *logrus.Entry) error { - if !vc.IsVacuumEnabled(logE) { - return nil - } - logE.Debug("closing vacuum controller") - if vc.d.storeQueue != nil { - vc.d.storeQueue.close() - } - return vc.d.Close() -} - -func (vc *Controller) TestKeepDBOpen() error { - return vc.d.TestKeepDBOpen() -} diff --git a/pkg/controller/vacuum/vacuum.go b/pkg/controller/vacuum/vacuum.go index b66e71a83..03a9b60be 100644 --- a/pkg/controller/vacuum/vacuum.go +++ b/pkg/controller/vacuum/vacuum.go @@ -54,6 +54,59 @@ func (vc *Controller) ListPackages(ctx context.Context, logE *logrus.Entry, expi return vc.handleListPackages(ctx, logE, args...) } +// Close closes the dependencies of the Controller. +func (vc *Controller) Close(logE *logrus.Entry) error { + if !vc.IsVacuumEnabled(logE) { + return nil + } + logE.Debug("closing vacuum controller") + if vc.d.storeQueue != nil { + vc.d.storeQueue.close() + } + return vc.d.Close() +} + +func (vc *Controller) TestKeepDBOpen() error { + return vc.d.TestKeepDBOpen() +} + +// StorePackage stores the given package if vacuum is enabled. +// If the package is nil, it logs a warning and skips storing the package. +func (vc *Controller) StorePackage(logE *logrus.Entry, pkg *config.Package, pkgPath string) error { + if !vc.IsVacuumEnabled(logE) { + return nil + } + if pkg == nil { + logE.Warn("package is nil, skipping store package") + return nil + } + return vc.handleAsyncStorePackage(logE, vc.getVacuumPackage(pkg, pkgPath)) +} + +// IsVacuumEnabled checks if the vacuum feature is enabled based on the configuration. +func (vc *Controller) IsVacuumEnabled(logE *logrus.Entry) bool { + if vc.Param.VacuumDays <= 0 { + logE.Debug("vacuum is disabled. AQUA_VACUUM_DAYS is not set or invalid.") + return false + } + return true +} + +// GetPackageLastUsed retrieves the last used time of a package. for testing purposes. +func (vc *Controller) GetPackageLastUsed(ctx context.Context, logE *logrus.Entry, pkgPath string) *time.Time { + var lastUsedTime time.Time + pkgEntry, _ := vc.retrievePackageEntry(ctx, logE, pkgPath) + if pkgEntry != nil { + lastUsedTime = pkgEntry.LastUsageTime + } + return &lastUsedTime +} + +// SetTimeStampPackage permit define a Timestamp for a package Manually. for testing purposes. +func (vc *Controller) SetTimestampPackage(ctx context.Context, logE *logrus.Entry, pkg *config.Package, pkgPath string, datetime time.Time) error { + return vc.d.Store(ctx, logE, vc.getVacuumPackage(pkg, pkgPath), datetime) +} + // handleListPackages retrieves a list of packages and displays them using a fuzzy search. func (vc *Controller) handleListPackages(ctx context.Context, logE *logrus.Entry, args ...string) error { pkgs, err := vc.listPackages(ctx, logE) @@ -73,19 +126,6 @@ func (vc *Controller) handleListExpiredPackages(ctx context.Context, logE *logru return vc.displayPackagesFuzzy(logE, expiredPkgs, args...) } -// StorePackage stores the given package if vacuum is enabled. -// If the package is nil, it logs a warning and skips storing the package. -func (vc *Controller) StorePackage(logE *logrus.Entry, pkg *config.Package, pkgPath string) error { - if !vc.IsVacuumEnabled(logE) { - return nil - } - if pkg == nil { - logE.Warn("package is nil, skipping store package") - return nil - } - return vc.handleAsyncStorePackage(logE, vc.getVacuumPackage(pkg, pkgPath)) -} - // getVacuumPackage converts a config func (vc *Controller) getVacuumPackage(configPkg *config.Package, pkgPath string) *Package { return &Package{ @@ -105,15 +145,6 @@ func (vc *Controller) handleAsyncStorePackage(logE *logrus.Entry, vacuumPkg *Pac return nil } -// IsVacuumEnabled checks if the vacuum feature is enabled based on the configuration. -func (vc *Controller) IsVacuumEnabled(logE *logrus.Entry) bool { - if vc.Param.VacuumDays <= 0 { - logE.Debug("vacuum is disabled. AQUA_VACUUM_DAYS is not set or invalid.") - return false - } - return true -} - // listExpiredPackages lists all packages that have expired based on the vacuum configuration. func (vc *Controller) listExpiredPackages(ctx context.Context, logE *logrus.Entry) ([]*PackageVacuumEntry, error) { pkgs, err := vc.listPackages(ctx, logE) @@ -154,7 +185,6 @@ func (vc *Controller) listPackages(ctx context.Context, logE *logrus.Entry) ([]* if db == nil { return nil, nil } - defer db.Close() var pkgs []*PackageVacuumEntry @@ -282,7 +312,6 @@ func (vc *Controller) vacuumExpiredPackages(ctx context.Context, logE *logrus.En return gErr } - defer vc.Close(logE) if len(successfulRemovals) > 0 { if err := vc.removePackages(ctx, logE, successfulRemovals); err != nil { return fmt.Errorf("remove packages from database: %w", err) @@ -398,21 +427,6 @@ func decodePackageEntry(data []byte) (*PackageEntry, error) { return &pkgEntry, nil } -// GetPackageLastUsed retrieves the last used time of a package. for testing purposes. -func (vc *Controller) GetPackageLastUsed(ctx context.Context, logE *logrus.Entry, pkgPath string) *time.Time { - var lastUsedTime time.Time - pkgEntry, _ := vc.retrievePackageEntry(ctx, logE, pkgPath) - if pkgEntry != nil { - lastUsedTime = pkgEntry.LastUsageTime - } - return &lastUsedTime -} - -// SetTimeStampPackage permit define a Timestamp for a package Manually. for testing purposes. -func (vc *Controller) SetTimestampPackage(ctx context.Context, logE *logrus.Entry, pkg *config.Package, pkgPath string, datetime time.Time) error { - return vc.d.Store(ctx, logE, vc.getVacuumPackage(pkg, pkgPath), datetime) -} - // retrievePackageEntry retrieves a package entry from the database by key. for testing purposes. func (vc *Controller) retrievePackageEntry(ctx context.Context, logE *logrus.Entry, key string) (*PackageEntry, error) { var pkgEntry *PackageEntry diff --git a/pkg/controller/vacuum/vacuum_db.go b/pkg/controller/vacuum/vacuum_db.go index 6725eb183..8b7218ecd 100644 --- a/pkg/controller/vacuum/vacuum_db.go +++ b/pkg/controller/vacuum/vacuum_db.go @@ -115,8 +115,8 @@ func (d *DB) Close() error { d.dbMutex.Lock() defer d.dbMutex.Unlock() - if d.db.Load() != nil { - if err := d.db.Load().Close(); err != nil { + if db := d.db.Load(); db != nil { + if err := db.Close(); err != nil { return fmt.Errorf("close database: %w", err) } d.db.Store(nil) From 0f035e1706850316076590321720b88589abc59f Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 14:24:04 +0900 Subject: [PATCH 65/73] fix: stop updating timestamp twice --- pkg/controller/exec/exec.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/controller/exec/exec.go b/pkg/controller/exec/exec.go index 61ee7bf61..2062ad9be 100644 --- a/pkg/controller/exec/exec.go +++ b/pkg/controller/exec/exec.go @@ -102,9 +102,6 @@ func (c *Controller) install(ctx context.Context, logE *logrus.Entry, findResult }); err != nil { return fmt.Errorf("install the package: %w", err) } - if err := c.vacuum.StorePackage(logE, findResult.Package, findResult.PkgPath); err != nil { - logerr.WithError(logE, err).Error("store the package") - } return c.checkExePath(ctx, logE, findResult) } From b875c9cc576603d5e78330c5a0ddc276ab1cc2d4 Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 14:29:08 +0900 Subject: [PATCH 66/73] fix: suppress a lint error --- pkg/controller/exec/exec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/exec/exec.go b/pkg/controller/exec/exec.go index 2062ad9be..579df1fcd 100644 --- a/pkg/controller/exec/exec.go +++ b/pkg/controller/exec/exec.go @@ -18,7 +18,7 @@ import ( "github.com/suzuki-shunsuke/logrus-error/logerr" ) -func (c *Controller) Exec(ctx context.Context, logE *logrus.Entry, param *config.Param, exeName string, args ...string) (gErr error) { +func (c *Controller) Exec(ctx context.Context, logE *logrus.Entry, param *config.Param, exeName string, args ...string) (gErr error) { //nolint:cyclop logE = logE.WithField("exe_name", exeName) defer func() { if gErr != nil { From 45981e43e602625d69d7819aafd155011cdc8eb9 Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 14:53:56 +0900 Subject: [PATCH 67/73] refactor: separate code --- pkg/controller/vacuum/fuzzy.go | 90 ++++++++++++ pkg/controller/vacuum/show.go | 36 +++++ pkg/controller/vacuum/vacuum.go | 218 ++--------------------------- pkg/controller/vacuum/vacuum_db.go | 70 +++++++++ 4 files changed, 204 insertions(+), 210 deletions(-) create mode 100644 pkg/controller/vacuum/fuzzy.go create mode 100644 pkg/controller/vacuum/show.go diff --git a/pkg/controller/vacuum/fuzzy.go b/pkg/controller/vacuum/fuzzy.go new file mode 100644 index 000000000..a66a9df7b --- /dev/null +++ b/pkg/controller/vacuum/fuzzy.go @@ -0,0 +1,90 @@ +package vacuum + +import ( + "errors" + "fmt" + + "github.com/dustin/go-humanize" + "github.com/ktr0731/go-fuzzyfinder" + "github.com/sirupsen/logrus" +) + +func (vc *Controller) displayPackagesFuzzyTest(logE *logrus.Entry, pkgs []*PackageVacuumEntry) error { + var pkgInformations struct { + TotalPackages int + TotalExpired int + } + for _, pkg := range pkgs { + if vc.isPackageExpired(pkg) { + pkgInformations.TotalExpired++ + } + pkgInformations.TotalPackages++ + } + // Display log entry with informations for testing purposes + logE.WithFields(logrus.Fields{ + "TotalPackages": pkgInformations.TotalPackages, + "TotalExpired": pkgInformations.TotalExpired, + }).Info("Test mode: Displaying packages") + return nil +} + +func (vc *Controller) displayPackagesFuzzy(logE *logrus.Entry, pkgs []*PackageVacuumEntry, args ...string) error { + if len(pkgs) == 0 { + logE.Info("no packages to display") + return nil + } + if len(args) > 0 && args[0] == "test" { + return vc.displayPackagesFuzzyTest(logE, pkgs) + } + return vc.displayPackagesFuzzyInteractive(pkgs) +} + +func (vc *Controller) displayPackagesFuzzyInteractive(pkgs []*PackageVacuumEntry) error { + _, err := fuzzyfinder.Find(pkgs, func(i int) string { + var expiredString string + if vc.isPackageExpired(pkgs[i]) { + expiredString = "⌛ " + } + + return fmt.Sprintf("%s%s [%s]", + expiredString, + pkgs[i].PkgPath, + humanize.Time(pkgs[i].PackageEntry.LastUsageTime), + ) + }, + fuzzyfinder.WithPreviewWindow(func(i, _, _ int) string { + if i == -1 { + return "No package selected" + } + pkg := pkgs[i] + var expiredString string + if vc.isPackageExpired(pkg) { + expiredString = "Expired ⌛" + } + return fmt.Sprintf( + "Package Details:\n\n"+ + "%s \n"+ + "Type: %s\n"+ + "Package: %s\n"+ + "Version: %s\n\n"+ + "Last Used: %s\n"+ + "Last Used (exact): %s\n\n", + expiredString, + pkg.PackageEntry.Package.Type, + pkg.PackageEntry.Package.Name, + pkg.PackageEntry.Package.Version, + humanize.Time(pkg.PackageEntry.LastUsageTime), + pkg.PackageEntry.LastUsageTime.Format("2006-01-02 15:04:05"), + ) + }), + fuzzyfinder.WithHeader("Navigate through packages to display details"), + ) + if err != nil { + if errors.Is(err, fuzzyfinder.ErrAbort) { + return nil + } + return fmt.Errorf("display packages: %w", err) + } + + return nil +} diff --git a/pkg/controller/vacuum/show.go b/pkg/controller/vacuum/show.go new file mode 100644 index 000000000..dc37b23ce --- /dev/null +++ b/pkg/controller/vacuum/show.go @@ -0,0 +1,36 @@ +package vacuum + +import ( + "context" + + "github.com/sirupsen/logrus" +) + +// ListPackages lists the packages based on the provided arguments. +// If the expired flag is set to true, it lists the expired packages. +// Otherwise, it lists all packages. +func (vc *Controller) ListPackages(ctx context.Context, logE *logrus.Entry, expired bool, args ...string) error { + if expired { + return vc.handleListExpiredPackages(ctx, logE, args...) + } + return vc.handleListPackages(ctx, logE, args...) +} + +// handleListPackages retrieves a list of packages and displays them using a fuzzy search. +func (vc *Controller) handleListPackages(ctx context.Context, logE *logrus.Entry, args ...string) error { + pkgs, err := vc.d.List(ctx, logE) + if err != nil { + return err + } + return vc.displayPackagesFuzzy(logE, pkgs, args...) +} + +// handleListExpiredPackages handles the process of listing expired packages +// and displaying them using a fuzzy search. +func (vc *Controller) handleListExpiredPackages(ctx context.Context, logE *logrus.Entry, args ...string) error { + expiredPkgs, err := vc.listExpiredPackages(ctx, logE) + if err != nil { + return err + } + return vc.displayPackagesFuzzy(logE, expiredPkgs, args...) +} diff --git a/pkg/controller/vacuum/vacuum.go b/pkg/controller/vacuum/vacuum.go index 03a9b60be..b74e0f2b2 100644 --- a/pkg/controller/vacuum/vacuum.go +++ b/pkg/controller/vacuum/vacuum.go @@ -2,7 +2,6 @@ package vacuum import ( "context" - "encoding/json" "errors" "fmt" "path/filepath" @@ -10,11 +9,8 @@ import ( "time" "github.com/aquaproj/aqua/v2/pkg/config" - "github.com/dustin/go-humanize" - "github.com/ktr0731/go-fuzzyfinder" "github.com/sirupsen/logrus" "github.com/suzuki-shunsuke/logrus-error/logerr" - "go.etcd.io/bbolt" ) type PackageVacuumEntries []PackageVacuumEntry @@ -44,16 +40,6 @@ func (vc *Controller) Vacuum(ctx context.Context, logE *logrus.Entry) error { return vc.vacuumExpiredPackages(ctx, logE) } -// ListPackages lists the packages based on the provided arguments. -// If the expired flag is set to true, it lists the expired packages. -// Otherwise, it lists all packages. -func (vc *Controller) ListPackages(ctx context.Context, logE *logrus.Entry, expired bool, args ...string) error { - if expired { - return vc.handleListExpiredPackages(ctx, logE, args...) - } - return vc.handleListPackages(ctx, logE, args...) -} - // Close closes the dependencies of the Controller. func (vc *Controller) Close(logE *logrus.Entry) error { if !vc.IsVacuumEnabled(logE) { @@ -95,7 +81,7 @@ func (vc *Controller) IsVacuumEnabled(logE *logrus.Entry) bool { // GetPackageLastUsed retrieves the last used time of a package. for testing purposes. func (vc *Controller) GetPackageLastUsed(ctx context.Context, logE *logrus.Entry, pkgPath string) *time.Time { var lastUsedTime time.Time - pkgEntry, _ := vc.retrievePackageEntry(ctx, logE, pkgPath) + pkgEntry, _ := vc.d.Get(ctx, logE, pkgPath) if pkgEntry != nil { lastUsedTime = pkgEntry.LastUsageTime } @@ -107,25 +93,6 @@ func (vc *Controller) SetTimestampPackage(ctx context.Context, logE *logrus.Entr return vc.d.Store(ctx, logE, vc.getVacuumPackage(pkg, pkgPath), datetime) } -// handleListPackages retrieves a list of packages and displays them using a fuzzy search. -func (vc *Controller) handleListPackages(ctx context.Context, logE *logrus.Entry, args ...string) error { - pkgs, err := vc.listPackages(ctx, logE) - if err != nil { - return err - } - return vc.displayPackagesFuzzy(logE, pkgs, args...) -} - -// handleListExpiredPackages handles the process of listing expired packages -// and displaying them using a fuzzy search. -func (vc *Controller) handleListExpiredPackages(ctx context.Context, logE *logrus.Entry, args ...string) error { - expiredPkgs, err := vc.listExpiredPackages(ctx, logE) - if err != nil { - return err - } - return vc.displayPackagesFuzzy(logE, expiredPkgs, args...) -} - // getVacuumPackage converts a config func (vc *Controller) getVacuumPackage(configPkg *config.Package, pkgPath string) *Package { return &Package{ @@ -145,22 +112,6 @@ func (vc *Controller) handleAsyncStorePackage(logE *logrus.Entry, vacuumPkg *Pac return nil } -// listExpiredPackages lists all packages that have expired based on the vacuum configuration. -func (vc *Controller) listExpiredPackages(ctx context.Context, logE *logrus.Entry) ([]*PackageVacuumEntry, error) { - pkgs, err := vc.listPackages(ctx, logE) - if err != nil { - return nil, err - } - - var expired []*PackageVacuumEntry - for _, pkg := range pkgs { - if vc.isPackageExpired(pkg) { - expired = append(expired, pkg) - } - } - return expired, nil -} - const secondsInADay = 24 * 60 * 60 // isPackageExpired checks if a package is expired based on the vacuum configuration. @@ -176,117 +127,20 @@ func (vc *Controller) isPackageExpired(pkg *PackageVacuumEntry) bool { return timeSinceLastUsage > float64(threshold) } -// listPackages lists all stored package entries. -func (vc *Controller) listPackages(ctx context.Context, logE *logrus.Entry) ([]*PackageVacuumEntry, error) { - db, err := vc.d.getDB() +// listExpiredPackages lists all packages that have expired based on the vacuum configuration. +func (vc *Controller) listExpiredPackages(ctx context.Context, logE *logrus.Entry) ([]*PackageVacuumEntry, error) { + pkgs, err := vc.d.List(ctx, logE) if err != nil { return nil, err } - if db == nil { - return nil, nil - } - - var pkgs []*PackageVacuumEntry - err = vc.d.view(ctx, logE, func(tx *bbolt.Tx) error { - b := vc.d.Bucket(tx) - if b == nil { - return nil - } - return b.ForEach(func(k, value []byte) error { - pkgEntry, err := decodePackageEntry(value) - if err != nil { - logerr.WithError(logE, err).WithField("pkg_key", string(k)).Warn("unable to decode entry") - return err - } - pkgs = append(pkgs, &PackageVacuumEntry{ - PkgPath: append([]byte{}, k...), - PackageEntry: pkgEntry, - }) - return nil - }) - }) - return pkgs, err -} - -func (vc *Controller) displayPackagesFuzzyTest(logE *logrus.Entry, pkgs []*PackageVacuumEntry) error { - var pkgInformations struct { - TotalPackages int - TotalExpired int - } + var expired []*PackageVacuumEntry for _, pkg := range pkgs { if vc.isPackageExpired(pkg) { - pkgInformations.TotalExpired++ - } - pkgInformations.TotalPackages++ - } - // Display log entry with informations for testing purposes - logE.WithFields(logrus.Fields{ - "TotalPackages": pkgInformations.TotalPackages, - "TotalExpired": pkgInformations.TotalExpired, - }).Info("Test mode: Displaying packages") - return nil -} - -func (vc *Controller) displayPackagesFuzzy(logE *logrus.Entry, pkgs []*PackageVacuumEntry, args ...string) error { - if len(pkgs) == 0 { - logE.Info("no packages to display") - return nil - } - if len(args) > 0 && args[0] == "test" { - return vc.displayPackagesFuzzyTest(logE, pkgs) - } - return vc.displayPackagesFuzzyInteractive(pkgs) -} - -func (vc *Controller) displayPackagesFuzzyInteractive(pkgs []*PackageVacuumEntry) error { - _, err := fuzzyfinder.Find(pkgs, func(i int) string { - var expiredString string - if vc.isPackageExpired(pkgs[i]) { - expiredString = "⌛ " - } - - return fmt.Sprintf("%s%s [%s]", - expiredString, - pkgs[i].PkgPath, - humanize.Time(pkgs[i].PackageEntry.LastUsageTime), - ) - }, - fuzzyfinder.WithPreviewWindow(func(i, _, _ int) string { - if i == -1 { - return "No package selected" - } - pkg := pkgs[i] - var expiredString string - if vc.isPackageExpired(pkg) { - expiredString = "Expired ⌛" - } - return fmt.Sprintf( - "Package Details:\n\n"+ - "%s \n"+ - "Type: %s\n"+ - "Package: %s\n"+ - "Version: %s\n\n"+ - "Last Used: %s\n"+ - "Last Used (exact): %s\n\n", - expiredString, - pkg.PackageEntry.Package.Type, - pkg.PackageEntry.Package.Name, - pkg.PackageEntry.Package.Version, - humanize.Time(pkg.PackageEntry.LastUsageTime), - pkg.PackageEntry.LastUsageTime.Format("2006-01-02 15:04:05"), - ) - }), - fuzzyfinder.WithHeader("Navigate through packages to display details"), - ) - if err != nil { - if errors.Is(err, fuzzyfinder.ErrAbort) { - return nil + expired = append(expired, pkg) } - return fmt.Errorf("display packages: %w", err) } - - return nil + return expired, nil } // vacuumExpiredPackages performs cleanup of expired packages. @@ -313,7 +167,7 @@ func (vc *Controller) vacuumExpiredPackages(ctx context.Context, logE *logrus.En } if len(successfulRemovals) > 0 { - if err := vc.removePackages(ctx, logE, successfulRemovals); err != nil { + if err := vc.d.RemovePackages(ctx, logE, successfulRemovals); err != nil { return fmt.Errorf("remove packages from database: %w", err) } } @@ -383,24 +237,6 @@ func (vc *Controller) processExpiredPackages(logE *logrus.Entry, expired []*Pack return pathsToRemove, errors } -// removePackages removes package entries from the database. -func (vc *Controller) removePackages(ctx context.Context, logE *logrus.Entry, pkgs []string) error { - return vc.d.update(ctx, logE, func(tx *bbolt.Tx) error { - b := vc.d.Bucket(tx) - if b == nil { - return errors.New("bucket not found") - } - - for _, key := range pkgs { - if err := b.Delete([]byte(key)); err != nil { - return fmt.Errorf("delete package %s: %w", key, err) - } - logE.WithField("pkgKey", key).Info("removed package from vacuum database") - } - return nil - }) -} - // removePackageVersionPath removes the specified package version directory and its parent directory if it becomes empty. func (vc *Controller) removePackageVersionPath(param *config.Param, path string) error { if err := vc.fs.RemoveAll(filepath.Join(param.RootDir, path)); err != nil { @@ -408,41 +244,3 @@ func (vc *Controller) removePackageVersionPath(param *config.Param, path string) } return nil } - -// encodePackageEntry encodes a PackageEntry into a JSON byte slice. -func encodePackageEntry(pkgEntry *PackageEntry) ([]byte, error) { - data, err := json.Marshal(pkgEntry) - if err != nil { - return nil, fmt.Errorf("marshal package entry: %w", err) - } - return data, nil -} - -// decodePackageEntry decodes a JSON byte slice into a PackageEntry. -func decodePackageEntry(data []byte) (*PackageEntry, error) { - var pkgEntry PackageEntry - if err := json.Unmarshal(data, &pkgEntry); err != nil { - return nil, fmt.Errorf("unmarshal package entry: %w", err) - } - return &pkgEntry, nil -} - -// retrievePackageEntry retrieves a package entry from the database by key. for testing purposes. -func (vc *Controller) retrievePackageEntry(ctx context.Context, logE *logrus.Entry, key string) (*PackageEntry, error) { - var pkgEntry *PackageEntry - err := vc.d.view(ctx, logE, func(tx *bbolt.Tx) error { - b := vc.d.Bucket(tx) - if b == nil { - return nil - } - value := b.Get([]byte(key)) - if value == nil { - return nil - } - - var err error - pkgEntry, err = decodePackageEntry(value) - return err - }) - return pkgEntry, err -} diff --git a/pkg/controller/vacuum/vacuum_db.go b/pkg/controller/vacuum/vacuum_db.go index 8b7218ecd..c24bece99 100644 --- a/pkg/controller/vacuum/vacuum_db.go +++ b/pkg/controller/vacuum/vacuum_db.go @@ -2,6 +2,7 @@ package vacuum import ( "context" + "encoding/json" "errors" "fmt" "io" @@ -110,6 +111,39 @@ func (d *DB) Store(ctx context.Context, logE *logrus.Entry, pkg *Package, lastUs }) } +// List lists all stored package entries. +func (d *DB) List(ctx context.Context, logE *logrus.Entry) ([]*PackageVacuumEntry, error) { + db, err := d.getDB() + if err != nil { + return nil, err + } + if db == nil { + return nil, nil + } + + var pkgs []*PackageVacuumEntry + + err = d.view(ctx, logE, func(tx *bbolt.Tx) error { + b := d.Bucket(tx) + if b == nil { + return nil + } + return b.ForEach(func(k, value []byte) error { + pkgEntry, err := decodePackageEntry(value) + if err != nil { + logerr.WithError(logE, err).WithField("pkg_key", string(k)).Warn("unable to decode entry") + return err + } + pkgs = append(pkgs, &PackageVacuumEntry{ + PkgPath: append([]byte{}, k...), + PackageEntry: pkgEntry, + }) + return nil + }) + }) + return pkgs, err +} + // Close closes the database instance. func (d *DB) Close() error { d.dbMutex.Lock() @@ -136,6 +170,24 @@ func (d *DB) TestKeepDBOpen() error { return nil } +// RemovePackages removes package entries from the database. +func (d *DB) RemovePackages(ctx context.Context, logE *logrus.Entry, pkgs []string) error { + return d.update(ctx, logE, func(tx *bbolt.Tx) error { + b := d.Bucket(tx) + if b == nil { + return errors.New("bucket not found") + } + + for _, key := range pkgs { + if err := b.Delete([]byte(key)); err != nil { + return fmt.Errorf("delete package %s: %w", key, err) + } + logE.WithField("pkg_key", key).Info("removed package from vacuum database") + } + return nil + }) +} + func (d *DB) view(ctx context.Context, logE *logrus.Entry, fn func(*bbolt.Tx) error) error { return d.withDBRetry(ctx, logE, fn, View) } @@ -231,3 +283,21 @@ func (d *DB) getDB() (*bbolt.DB, error) { d.db.Store(db) return db, nil } + +// encodePackageEntry encodes a PackageEntry into a JSON byte slice. +func encodePackageEntry(pkgEntry *PackageEntry) ([]byte, error) { + data, err := json.Marshal(pkgEntry) + if err != nil { + return nil, fmt.Errorf("marshal package entry: %w", err) + } + return data, nil +} + +// decodePackageEntry decodes a JSON byte slice into a PackageEntry. +func decodePackageEntry(data []byte) (*PackageEntry, error) { + var pkgEntry PackageEntry + if err := json.Unmarshal(data, &pkgEntry); err != nil { + return nil, fmt.Errorf("unmarshal package entry: %w", err) + } + return &pkgEntry, nil +} From 4771a9000331283789b8188b3e72bcd620b25555 Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 15:27:11 +0900 Subject: [PATCH 68/73] fix(exec): fix a bug that DB isn't closed --- pkg/cli/exec/command.go | 1 - pkg/controller/exec/exec.go | 14 +++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/cli/exec/command.go b/pkg/cli/exec/command.go index b73d875e8..4c2a298e4 100644 --- a/pkg/cli/exec/command.go +++ b/pkg/cli/exec/command.go @@ -56,6 +56,5 @@ func (i *command) action(c *cli.Context) error { if err != nil { return fmt.Errorf("parse args: %w", err) } - defer ctrl.CloseVacuum(logE) return ctrl.Exec(c.Context, logE, param, exeName, args...) //nolint:wrapcheck } diff --git a/pkg/controller/exec/exec.go b/pkg/controller/exec/exec.go index 579df1fcd..4baa09c1c 100644 --- a/pkg/controller/exec/exec.go +++ b/pkg/controller/exec/exec.go @@ -44,10 +44,6 @@ func (c *Controller) Exec(ctx context.Context, logE *logrus.Entry, param *config return c.execCommandWithRetry(ctx, logE, findResult.ExePath, args...) } - if err := c.vacuum.StorePackage(logE, findResult.Package, findResult.PkgPath); err != nil { - logerr.WithError(logE, err).Error("store the package") - } - logE = logE.WithFields(logrus.Fields{ "package_name": findResult.Package.Package.Name, "package_version": findResult.Package.Package.Version, @@ -63,13 +59,21 @@ func (c *Controller) Exec(ctx context.Context, logE *logrus.Entry, param *config return logerr.WithFields(errExecNotFoundDisableLazyInstall, logE.WithField("doc", "https://aquaproj.github.io/docs/reference/codes/006").Data) //nolint:wrapcheck } } + + if err := c.vacuum.StorePackage(logE, findResult.Package, findResult.PkgPath); err != nil { + logerr.WithError(logE, err).Error("store the package") + } + + defer c.closeVacuum(logE) if err := c.install(ctx, logE, findResult, policyCfgs, param); err != nil { return err } + c.closeVacuum(logE) + return c.execCommandWithRetry(ctx, logE, findResult.ExePath, args...) } -func (c *Controller) CloseVacuum(logE *logrus.Entry) { +func (c *Controller) closeVacuum(logE *logrus.Entry) { if err := c.vacuum.Close(logE); err != nil { logerr.WithError(logE, err).Error("close the vacuum") } From f815a6a3cc9400651e39d67ac8775053f1a18267 Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 19:10:51 +0900 Subject: [PATCH 69/73] style: sort methods --- pkg/controller/vacuum/vacuum_db.go | 76 +++++++++++++++--------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/pkg/controller/vacuum/vacuum_db.go b/pkg/controller/vacuum/vacuum_db.go index c24bece99..2728b6ee0 100644 --- a/pkg/controller/vacuum/vacuum_db.go +++ b/pkg/controller/vacuum/vacuum_db.go @@ -53,26 +53,6 @@ func (d *DB) Bucket(tx *bbolt.Tx) *bbolt.Bucket { return tx.Bucket([]byte(bucketNamePkgs)) } -// Get retrieves a package entry from the database by key. for testing purposes. -func (d *DB) Get(ctx context.Context, logE *logrus.Entry, key string) (*PackageEntry, error) { - var pkgEntry *PackageEntry - err := d.withDBRetry(ctx, logE, func(tx *bbolt.Tx) error { - b := tx.Bucket([]byte(bucketNamePkgs)) - if b == nil { - return nil - } - value := b.Get([]byte(key)) - if value == nil { - return nil - } - - var err error - pkgEntry, err = decodePackageEntry(value) - return err - }, View) - return pkgEntry, err -} - // Store stores package entries in the database. func (d *DB) Store(ctx context.Context, logE *logrus.Entry, pkg *Package, lastUsedTime time.Time) error { return d.update(ctx, logE, func(tx *bbolt.Tx) error { @@ -144,6 +124,24 @@ func (d *DB) List(ctx context.Context, logE *logrus.Entry) ([]*PackageVacuumEntr return pkgs, err } +// RemovePackages removes package entries from the database. +func (d *DB) RemovePackages(ctx context.Context, logE *logrus.Entry, pkgs []string) error { + return d.update(ctx, logE, func(tx *bbolt.Tx) error { + b := d.Bucket(tx) + if b == nil { + return errors.New("bucket not found") + } + + for _, key := range pkgs { + if err := b.Delete([]byte(key)); err != nil { + return fmt.Errorf("delete package %s: %w", key, err) + } + logE.WithField("pkg_key", key).Info("removed package from vacuum database") + } + return nil + }) +} + // Close closes the database instance. func (d *DB) Close() error { d.dbMutex.Lock() @@ -159,6 +157,26 @@ func (d *DB) Close() error { return nil } +// Get retrieves a package entry from the database by key. for testing purposes. +func (d *DB) Get(ctx context.Context, logE *logrus.Entry, key string) (*PackageEntry, error) { + var pkgEntry *PackageEntry + err := d.withDBRetry(ctx, logE, func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(bucketNamePkgs)) + if b == nil { + return nil + } + value := b.Get([]byte(key)) + if value == nil { + return nil + } + + var err error + pkgEntry, err = decodePackageEntry(value) + return err + }, View) + return pkgEntry, err +} + // TesetKeepDBOpen opens the database instance. This is used for testing purposes. func (d *DB) TestKeepDBOpen() error { const dbFileMode = 0o600 @@ -170,24 +188,6 @@ func (d *DB) TestKeepDBOpen() error { return nil } -// RemovePackages removes package entries from the database. -func (d *DB) RemovePackages(ctx context.Context, logE *logrus.Entry, pkgs []string) error { - return d.update(ctx, logE, func(tx *bbolt.Tx) error { - b := d.Bucket(tx) - if b == nil { - return errors.New("bucket not found") - } - - for _, key := range pkgs { - if err := b.Delete([]byte(key)); err != nil { - return fmt.Errorf("delete package %s: %w", key, err) - } - logE.WithField("pkg_key", key).Info("removed package from vacuum database") - } - return nil - }) -} - func (d *DB) view(ctx context.Context, logE *logrus.Entry, fn func(*bbolt.Tx) error) error { return d.withDBRetry(ctx, logE, fn, View) } From 8eb89a0491cacf9836532e656cacd827adbd690d Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 21:49:28 +0900 Subject: [PATCH 70/73] fix: remove package type from db --- pkg/controller/vacuum/fuzzy.go | 2 -- pkg/controller/vacuum/vacuum.go | 2 -- 2 files changed, 4 deletions(-) diff --git a/pkg/controller/vacuum/fuzzy.go b/pkg/controller/vacuum/fuzzy.go index a66a9df7b..3711e2cad 100644 --- a/pkg/controller/vacuum/fuzzy.go +++ b/pkg/controller/vacuum/fuzzy.go @@ -64,13 +64,11 @@ func (vc *Controller) displayPackagesFuzzyInteractive(pkgs []*PackageVacuumEntry return fmt.Sprintf( "Package Details:\n\n"+ "%s \n"+ - "Type: %s\n"+ "Package: %s\n"+ "Version: %s\n\n"+ "Last Used: %s\n"+ "Last Used (exact): %s\n\n", expiredString, - pkg.PackageEntry.Package.Type, pkg.PackageEntry.Package.Name, pkg.PackageEntry.Package.Version, humanize.Time(pkg.PackageEntry.LastUsageTime), diff --git a/pkg/controller/vacuum/vacuum.go b/pkg/controller/vacuum/vacuum.go index b74e0f2b2..1641853e0 100644 --- a/pkg/controller/vacuum/vacuum.go +++ b/pkg/controller/vacuum/vacuum.go @@ -26,7 +26,6 @@ type PackageEntry struct { } type Package struct { - Type string // Type of package (e.g. "github_release") Name string // Name of package (e.g. "cli/cli") Version string // Version of package (e.g. "v1.0.0") PkgPath string // Path to the install path without the rootDir/pkgs/ prefix @@ -96,7 +95,6 @@ func (vc *Controller) SetTimestampPackage(ctx context.Context, logE *logrus.Entr // getVacuumPackage converts a config func (vc *Controller) getVacuumPackage(configPkg *config.Package, pkgPath string) *Package { return &Package{ - Type: configPkg.PackageInfo.Type, Name: configPkg.Package.Name, Version: configPkg.Package.Version, PkgPath: pkgPath, From feb29c23a8ac4d81d4ee152bf7422b6fb899b49c Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 21:56:39 +0900 Subject: [PATCH 71/73] refactor: rename a field --- pkg/controller/vacuum/controller.go | 4 ++-- pkg/controller/vacuum/show.go | 2 +- pkg/controller/vacuum/vacuum.go | 18 +++++++++--------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/controller/vacuum/controller.go b/pkg/controller/vacuum/controller.go index 8f705923a..a2495215f 100644 --- a/pkg/controller/vacuum/controller.go +++ b/pkg/controller/vacuum/controller.go @@ -13,7 +13,7 @@ type Controller struct { stdout io.Writer Param *config.Param fs afero.Fs - d *DB + db *DB } // New initializes a Controller with the given context, parameters, and dependencies. @@ -22,7 +22,7 @@ func New(ctx context.Context, param *config.Param, fs afero.Fs) *Controller { stdout: os.Stdout, Param: param, fs: fs, - d: NewDB(ctx, param, fs), + db: NewDB(ctx, param, fs), } return vc } diff --git a/pkg/controller/vacuum/show.go b/pkg/controller/vacuum/show.go index dc37b23ce..d0dc6eb4d 100644 --- a/pkg/controller/vacuum/show.go +++ b/pkg/controller/vacuum/show.go @@ -18,7 +18,7 @@ func (vc *Controller) ListPackages(ctx context.Context, logE *logrus.Entry, expi // handleListPackages retrieves a list of packages and displays them using a fuzzy search. func (vc *Controller) handleListPackages(ctx context.Context, logE *logrus.Entry, args ...string) error { - pkgs, err := vc.d.List(ctx, logE) + pkgs, err := vc.db.List(ctx, logE) if err != nil { return err } diff --git a/pkg/controller/vacuum/vacuum.go b/pkg/controller/vacuum/vacuum.go index 1641853e0..25abbd50b 100644 --- a/pkg/controller/vacuum/vacuum.go +++ b/pkg/controller/vacuum/vacuum.go @@ -45,14 +45,14 @@ func (vc *Controller) Close(logE *logrus.Entry) error { return nil } logE.Debug("closing vacuum controller") - if vc.d.storeQueue != nil { - vc.d.storeQueue.close() + if vc.db.storeQueue != nil { + vc.db.storeQueue.close() } - return vc.d.Close() + return vc.db.Close() } func (vc *Controller) TestKeepDBOpen() error { - return vc.d.TestKeepDBOpen() + return vc.db.TestKeepDBOpen() } // StorePackage stores the given package if vacuum is enabled. @@ -80,7 +80,7 @@ func (vc *Controller) IsVacuumEnabled(logE *logrus.Entry) bool { // GetPackageLastUsed retrieves the last used time of a package. for testing purposes. func (vc *Controller) GetPackageLastUsed(ctx context.Context, logE *logrus.Entry, pkgPath string) *time.Time { var lastUsedTime time.Time - pkgEntry, _ := vc.d.Get(ctx, logE, pkgPath) + pkgEntry, _ := vc.db.Get(ctx, logE, pkgPath) if pkgEntry != nil { lastUsedTime = pkgEntry.LastUsageTime } @@ -89,7 +89,7 @@ func (vc *Controller) GetPackageLastUsed(ctx context.Context, logE *logrus.Entry // SetTimeStampPackage permit define a Timestamp for a package Manually. for testing purposes. func (vc *Controller) SetTimestampPackage(ctx context.Context, logE *logrus.Entry, pkg *config.Package, pkgPath string, datetime time.Time) error { - return vc.d.Store(ctx, logE, vc.getVacuumPackage(pkg, pkgPath), datetime) + return vc.db.Store(ctx, logE, vc.getVacuumPackage(pkg, pkgPath), datetime) } // getVacuumPackage converts a config @@ -106,7 +106,7 @@ func (vc *Controller) handleAsyncStorePackage(logE *logrus.Entry, vacuumPkg *Pac if vacuumPkg == nil { return errors.New("vacuumPkg is nil") } - vc.d.storeQueue.enqueue(logE, vacuumPkg) + vc.db.storeQueue.enqueue(logE, vacuumPkg) return nil } @@ -127,7 +127,7 @@ func (vc *Controller) isPackageExpired(pkg *PackageVacuumEntry) bool { // listExpiredPackages lists all packages that have expired based on the vacuum configuration. func (vc *Controller) listExpiredPackages(ctx context.Context, logE *logrus.Entry) ([]*PackageVacuumEntry, error) { - pkgs, err := vc.d.List(ctx, logE) + pkgs, err := vc.db.List(ctx, logE) if err != nil { return nil, err } @@ -165,7 +165,7 @@ func (vc *Controller) vacuumExpiredPackages(ctx context.Context, logE *logrus.En } if len(successfulRemovals) > 0 { - if err := vc.d.RemovePackages(ctx, logE, successfulRemovals); err != nil { + if err := vc.db.RemovePackages(ctx, logE, successfulRemovals); err != nil { return fmt.Errorf("remove packages from database: %w", err) } } From 0dadc3032bd304069932f8c2ffd8143b92460118 Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 21:57:39 +0900 Subject: [PATCH 72/73] refactor: remove an unused field --- pkg/controller/vacuum/controller.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/pkg/controller/vacuum/controller.go b/pkg/controller/vacuum/controller.go index a2495215f..e9520d582 100644 --- a/pkg/controller/vacuum/controller.go +++ b/pkg/controller/vacuum/controller.go @@ -2,27 +2,23 @@ package vacuum import ( "context" - "io" - "os" "github.com/aquaproj/aqua/v2/pkg/config" "github.com/spf13/afero" ) type Controller struct { - stdout io.Writer - Param *config.Param - fs afero.Fs - db *DB + Param *config.Param + fs afero.Fs + db *DB } // New initializes a Controller with the given context, parameters, and dependencies. func New(ctx context.Context, param *config.Param, fs afero.Fs) *Controller { vc := &Controller{ - stdout: os.Stdout, - Param: param, - fs: fs, - db: NewDB(ctx, param, fs), + Param: param, + fs: fs, + db: NewDB(ctx, param, fs), } return vc } From 337188f60399da030748d36ef898f7e4f149bc8a Mon Sep 17 00:00:00 2001 From: Shunsuke Suzuki Date: Tue, 21 Jan 2025 23:31:51 +0900 Subject: [PATCH 73/73] refactor: refactor --- pkg/controller/vacuum/queue_store.go | 4 ++-- pkg/controller/vacuum/vacuum.go | 2 +- pkg/controller/vacuum/vacuum_db.go | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/controller/vacuum/queue_store.go b/pkg/controller/vacuum/queue_store.go index da1e6fb74..e5834d217 100644 --- a/pkg/controller/vacuum/queue_store.go +++ b/pkg/controller/vacuum/queue_store.go @@ -67,8 +67,8 @@ func (sq *StoreQueue) worker(ctx context.Context) { } } -// enqueue adds a task to the queue. -func (sq *StoreQueue) enqueue(logE *logrus.Entry, pkg *Package) { +// Enqueue adds a task to the queue. +func (sq *StoreQueue) Enqueue(logE *logrus.Entry, pkg *Package) { select { case <-sq.done: return diff --git a/pkg/controller/vacuum/vacuum.go b/pkg/controller/vacuum/vacuum.go index 25abbd50b..4f8c6a854 100644 --- a/pkg/controller/vacuum/vacuum.go +++ b/pkg/controller/vacuum/vacuum.go @@ -106,7 +106,7 @@ func (vc *Controller) handleAsyncStorePackage(logE *logrus.Entry, vacuumPkg *Pac if vacuumPkg == nil { return errors.New("vacuumPkg is nil") } - vc.db.storeQueue.enqueue(logE, vacuumPkg) + vc.db.StoreAsync(logE, vacuumPkg) return nil } diff --git a/pkg/controller/vacuum/vacuum_db.go b/pkg/controller/vacuum/vacuum_db.go index 2728b6ee0..5dd2bd16c 100644 --- a/pkg/controller/vacuum/vacuum_db.go +++ b/pkg/controller/vacuum/vacuum_db.go @@ -91,6 +91,10 @@ func (d *DB) Store(ctx context.Context, logE *logrus.Entry, pkg *Package, lastUs }) } +func (d *DB) StoreAsync(logE *logrus.Entry, pkg *Package) { + d.storeQueue.Enqueue(logE, pkg) +} + // List lists all stored package entries. func (d *DB) List(ctx context.Context, logE *logrus.Entry) ([]*PackageVacuumEntry, error) { db, err := d.getDB()