Caching mechanism and streaming

.gitignore

@@ -22,3 +22,5 @@ super-linter.log
 demo/
 _site/
 .vscode/
+*.log.*
+*.log

.vscode/launch.json

@@ -36,9 +36,10 @@
       "envFile": "${workspaceFolder}/.env",
       "args": [
         "test",
-        "unzip",
-        "--zip-file=/Volumes/local_storage_m2/test.pvm",
-        "--destination=/Volumes/local_storage_m2/test_folder_devops",
+        "push-file",
+        "--file_path=/Users/cjlapao/Downloads",
+        "--target_path=dropbox/test_machine/macos",
+        "--target_filename=21de185744bf519e687cdf12f62b1c741371cdfa5e747b029056710e5b8c57fe-1.pvm"
       ]
     },
     {

go.work

@@ -1,3 +1,3 @@
-go 1.21
+go 1.22.0
 
 use ./src

go.work.sum

@@ -4,6 +4,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cjlapao/common-go-identity v0.0.3/go.mod h1:xuNepNCHVI/51Q6DQgNPYvx3HS0VaeEhGnp8YcDO/+I=
+github.com/cjlapao/common-go-logger v0.0.9 h1:ZFUs0tVOn7KydxOnDSPtz3TvksaOPiNxlRT2VcMQTLs=
+github.com/cjlapao/common-go-logger v0.0.9/go.mod h1:Ao96R8kuUfeTFY4lAhRFfTpnlb8F5eO7aThI5nzCTzA=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/go-jose/go-jose/v4 v4.0.1/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY=

src/basecontext/main.go

@@ -4,13 +4,12 @@ import (
 	"context"
 	"net/http"
 
-	"github.com/Parallels/prl-devops-service/common"
 	"github.com/Parallels/prl-devops-service/constants"
 	"github.com/Parallels/prl-devops-service/models"
 	log "github.com/cjlapao/common-go-logger"
 )
 
-var Logger = log.Get().WithTimestamp()
+var logger = log.Get().WithTimestamp()
 
 type BaseContext struct {
 	shouldLog   bool
@@ -115,7 +114,11 @@ func (c *BaseContext) DisableLog() {
 }
 
 func (c *BaseContext) ToggleLogTimestamps(value bool) {
-	common.Logger.EnableTimestamp(value)
+	logger.EnableTimestamp(value)
+}
+
+func (c *BaseContext) EnableLogFile(filename string) {
+	logger.AddFileLogger(filename)
 }
 
 func (c *BaseContext) LogInfof(format string, a ...interface{}) {
@@ -130,7 +133,7 @@ func (c *BaseContext) LogInfof(format string, a ...interface{}) {
 	}
 
 	msg += format
-	common.Logger.Info(msg, a...)
+	logger.Info(msg, a...)
 }
 
 func (c *BaseContext) LogErrorf(format string, a ...interface{}) {
@@ -144,7 +147,7 @@ func (c *BaseContext) LogErrorf(format string, a ...interface{}) {
 		msg = "[" + c.GetRequestId() + "] "
 	}
 	msg += format
-	common.Logger.Error(msg, a...)
+	logger.Error(msg, a...)
 }
 
 func (c *BaseContext) LogDebugf(format string, a ...interface{}) {
@@ -157,7 +160,7 @@ func (c *BaseContext) LogDebugf(format string, a ...interface{}) {
 		msg = "[" + c.GetRequestId() + "] "
 	}
 	msg += format
-	common.Logger.Debug(msg, a...)
+	logger.Debug(msg, a...)
 }
 
 func (c *BaseContext) LogWarnf(format string, a ...interface{}) {
@@ -171,7 +174,7 @@ func (c *BaseContext) LogWarnf(format string, a ...interface{}) {
 		msg = "[" + c.GetRequestId() + "] "
 	}
 	msg += format
-	common.Logger.Warn(msg, a...)
+	logger.Warn(msg, a...)
 }
 
 func (c *BaseContext) LogTracef(format string, a ...interface{}) {
@@ -185,5 +188,5 @@ func (c *BaseContext) LogTracef(format string, a ...interface{}) {
 		msg = "[" + c.GetRequestId() + "] "
 	}
 	msg += format
-	common.Logger.Trace(msg, a...)
+	logger.Trace(msg, a...)
 }

src/catalog/cacheservice/cache_file.go

@@ -0,0 +1 @@
+package cacheservice

src/catalog/cacheservice/cache_item_file.go

@@ -0,0 +1,25 @@
+package cacheservice
+
+type CacheItemFile struct {
+	BaseName         string
+	CacheFileName    string
+	IsCompressed     bool
+	IsCachedFolder   bool
+	NeedsRenaming    bool
+	MetadataFileName string
+	InvalidFiles     []string
+}
+
+func NewCacheItemFile(baseName string) *CacheItemFile {
+	return &CacheItemFile{
+		BaseName: baseName,
+	}
+}
+
+func (c *CacheItemFile) IsValid() bool {
+	return c.CacheFileName != "" && c.MetadataFileName != ""
+}
+
+func (c *CacheItemFile) NeedsCleaning() bool {
+	return len(c.InvalidFiles) > 0
+}

src/catalog/cacheservice/cleanup_requirements.go

@@ -0,0 +1,10 @@
+package cacheservice
+
+type CleanupRequirements struct {
+	NeedsCleaning    bool
+	SpaceNeeded      int64
+	IsFatal          bool
+	Reason           string
+	FreeDiskSpace    int64
+	CatalogTotalSize int64
+}