fix: Add smaller interval to metadata cleanup so the jobs are cleared…

… frequently
keboola · Nov 19, 2024 · c78fba5 · c78fba5
1 parent 38bea92
commit c78fba5
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 4 deletions.
diff --git a/internal/pkg/service/stream/config/config_test.go b/internal/pkg/service/stream/config/config_test.go
@@ -149,8 +149,8 @@ storage:
     metadataCleanup:
         # Enable local storage metadata cleanup.
         enabled: true
-        # Cleanup interval. Validation rules: required,minDuration=5m,maxDuration=24h
-        interval: 5m0s
+        # Cleanup interval. Validation rules: required,minDuration=30s,maxDuration=24h
+        interval: 30s
         # How many files are deleted in parallel. Validation rules: required,min=1,max=500
         concurrency: 50
         # Expiration interval of a file that has not yet been imported. Validation rules: required,minDuration=1h,maxDuration=720h,gtefield=ArchivedFileExpiration

diff --git a/internal/pkg/service/stream/storage/metacleanup/config.go b/internal/pkg/service/stream/storage/metacleanup/config.go
@@ -4,7 +4,7 @@ import "time"
 
 type Config struct {
 	Enabled                bool          `configKey:"enabled"  configUsage:"Enable local storage metadata cleanup."`
-	Interval               time.Duration `configKey:"interval"  configUsage:"Cleanup interval." validate:"required,minDuration=5m,maxDuration=24h"`
+	Interval               time.Duration `configKey:"interval"  configUsage:"Cleanup interval." validate:"required,minDuration=30s,maxDuration=24h"`
 	Concurrency            int           `configKey:"concurrency"  configUsage:"How many files are deleted in parallel." validate:"required,min=1,max=500"`
 	ActiveFileExpiration   time.Duration `configKey:"activeFileExpiration"  configUsage:"Expiration interval of a file that has not yet been imported." validate:"required,minDuration=1h,maxDuration=720h,gtefield=ArchivedFileExpiration"` // maxDuration=30 days
 	ArchivedFileExpiration time.Duration `configKey:"archivedFileExpiration"  configUsage:"Expiration interval of a file that has already been imported." validate:"required,minDuration=15m,maxDuration=720h"`                              // maxDuration=30 days
@@ -13,7 +13,7 @@ type Config struct {
 func NewConfig() Config {
 	return Config{
 		Enabled:                true,
-		Interval:               5 * time.Minute,
+		Interval:               30 * time.Second,
 		Concurrency:            50,
 		ActiveFileExpiration:   7 * 24 * time.Hour, // 7 days
 		ArchivedFileExpiration: 6 * time.Hour,