From dd6787fe08fa271a9f88d7ccf0a2285f6cdcae6d Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 6 Sep 2024 14:41:29 +0530 Subject: [PATCH 1/9] add slash seperator --- code/go/0chain.net/blobbercore/reference/referencepath.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 16b0e6884..f9566a1b8 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -264,7 +264,7 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, } dbQuery = dbQuery.Order("path") } else { - dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND path LIKE ?", allocationID, path+"%") + dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND path LIKE ?", allocationID, path+"/%") if _type != "" { dbQuery = dbQuery.Where("type = ?", _type) } From b36c70770a72377eb29a4a3337905bee563f2262 Mon Sep 17 00:00:00 2001 From: shahnawaz-creator <117025384+shahnawaz-creator@users.noreply.github.com> Date: Fri, 6 Sep 2024 19:50:53 +0530 Subject: [PATCH 2/9] fix build-&-publish-docker-image.yml --- .../build-&-publish-docker-image.yml | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index dbd46074d..5dc567d83 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -56,10 +56,13 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} + run: | + mkdir -p /tmp/docker-config-${{ github.run_id }} + docker --config /tmp/docker-config-${{ github.run_id }} login -u ${{ secrets.DOCKERHUB_USERNAME }} -p ${{ secrets.DOCKERHUB_PASSWORD }} + # uses: docker/login-action@v1 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_PASSWORD }} # - name: Get changed files using defaults # id: changed-files @@ -96,6 +99,10 @@ jobs: docker tag ${BLOBBER_REGISTRY}:${TAG} ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA} docker push ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA} + - name: Clean up Docker Config + run: | + rm -rf /tmp/docker-config-${{ github.run_id }} + validator: timeout-minutes: 30 runs-on: [blobber-runner] @@ -136,11 +143,16 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} + run: | + mkdir -p /tmp/docker-config-${{ github.run_id }} + docker --config /tmp/docker-config-${{ github.run_id }} login -u ${{ secrets.DOCKERHUB_USERNAME }} -p ${{ secrets.DOCKERHUB_PASSWORD }} + # uses: docker/login-action@v1 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_PASSWORD }} + # - name: Get changed files using defaults # id: changed-files @@ -178,6 +190,10 @@ jobs: docker tag ${VALIDATOR_REGISTRY}:${TAG} ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA} docker push ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA} + - name: Clean up Docker Config + run: | + rm -rf /tmp/docker-config-${{ github.run_id }} + system-tests: if: github.event_name != 'workflow_dispatch' needs: [blobber, validator] From fd6145e07ba689976f36e1797de56ff56f1c67bd Mon Sep 17 00:00:00 2001 From: M-S-A-C Date: Tue, 17 Sep 2024 14:55:56 +0530 Subject: [PATCH 3/9] fixed dockerfile to use uppercase for AS --- docker.local/blobber.Dockerfile | 2 +- docker.local/validator.Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker.local/blobber.Dockerfile b/docker.local/blobber.Dockerfile index fdfb40ee8..c66835945 100644 --- a/docker.local/blobber.Dockerfile +++ b/docker.local/blobber.Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 ARG DOCKER_IMAGE_BASE -FROM $DOCKER_IMAGE_BASE as blobber_build +FROM $DOCKER_IMAGE_BASE AS blobber_build LABEL zchain="blobber" ENV SRC_DIR=/0chain diff --git a/docker.local/validator.Dockerfile b/docker.local/validator.Dockerfile index 7c1313cc6..0ae0c0628 100644 --- a/docker.local/validator.Dockerfile +++ b/docker.local/validator.Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 ARG DOCKER_IMAGE_BASE -FROM $DOCKER_IMAGE_BASE as validator_build +FROM $DOCKER_IMAGE_BASE AS validator_build LABEL zchain="validator" From 350bb7ef03ab7b751c7273ac2a282c44f7d67827 Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Tue, 24 Sep 2024 16:21:24 +0530 Subject: [PATCH 4/9] fix list root --- code/go/0chain.net/blobbercore/reference/referencepath.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index f9566a1b8..8972d50ec 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -264,7 +264,11 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, } dbQuery = dbQuery.Order("path") } else { - dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND path LIKE ?", allocationID, path+"/%") + listPath := path + if path != "/" { + listPath = path + "/" + } + dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND path LIKE ?", allocationID, listPath+"%") if _type != "" { dbQuery = dbQuery.Where("type = ?", _type) } From bc448fcd778933fa879724af5fd97f068cc4d332 Mon Sep 17 00:00:00 2001 From: shahnawaz-creator <117025384+shahnawaz-creator@users.noreply.github.com> Date: Tue, 24 Sep 2024 17:26:23 +0530 Subject: [PATCH 5/9] fix build-&-publish-docker-image.yml --- .github/workflows/build-&-publish-docker-image.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index 5dc567d83..9bfb20263 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -47,6 +47,12 @@ jobs: # with: # go-version: ^1.21 # The Go version to download (if necessary) and use. + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Clone blobber uses: actions/checkout@v3 with: @@ -59,10 +65,6 @@ jobs: run: | mkdir -p /tmp/docker-config-${{ github.run_id }} docker --config /tmp/docker-config-${{ github.run_id }} login -u ${{ secrets.DOCKERHUB_USERNAME }} -p ${{ secrets.DOCKERHUB_PASSWORD }} - # uses: docker/login-action@v1 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_PASSWORD }} # - name: Get changed files using defaults # id: changed-files From f8aefee12d0ab381fe25452876120dbed75246de Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 25 Sep 2024 00:21:14 +0530 Subject: [PATCH 6/9] add log for connection obj --- code/go/0chain.net/blobbercore/allocation/connection.go | 2 ++ code/go/0chain.net/blobbercore/handler/worker.go | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/code/go/0chain.net/blobbercore/allocation/connection.go b/code/go/0chain.net/blobbercore/allocation/connection.go index 75ac30b0c..47e418712 100644 --- a/code/go/0chain.net/blobbercore/allocation/connection.go +++ b/code/go/0chain.net/blobbercore/allocation/connection.go @@ -246,6 +246,7 @@ func GetHasher(connectionID, pathHash string) *filestore.CommitHasher { // DeleteConnectionObjEntry remove the connectionID entry from map // If the given connectionID is not present, then it is no-op. func DeleteConnectionObjEntry(connectionID string) { + logging.Logger.Info("DeleteConnectionObjEntry", zap.String("connection_id", connectionID)) connectionObjMutex.Lock() connectionObj, ok := connectionProcessor[connectionID] if ok { @@ -262,6 +263,7 @@ func cleanConnectionObj() { for connectionID, connectionObj := range connectionProcessor { diff := time.Since(connectionObj.UpdatedAt) if diff >= ConnectionObjTimeout { + logging.Logger.Info("cleanConnectionObj", zap.String("connection_id", connectionID), zap.Duration("diff", diff)) // Stop the context and hash worker connectionObj.cnclCtx() for _, change := range connectionObj.changes { diff --git a/code/go/0chain.net/blobbercore/handler/worker.go b/code/go/0chain.net/blobbercore/handler/worker.go index 19cf49c30..b5acbe174 100644 --- a/code/go/0chain.net/blobbercore/handler/worker.go +++ b/code/go/0chain.net/blobbercore/handler/worker.go @@ -87,6 +87,10 @@ func cleanupTempFiles(ctx context.Context) { for i := 0; i < len(openConnectionsToDelete); i++ { connection := &openConnectionsToDelete[i] logging.Logger.Info("Deleting temp files for the connection", zap.Any("connection", connection.ID)) + processor := allocation.GetConnectionProcessor(connection.ID) + if processor != nil { + continue + } connection.ComputeProperties() nctx := datastore.GetStore().CreateTransaction(ctx) From ffb861c68cb877d8cb8f9c2dfd34bd3e2f641cef Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 27 Sep 2024 13:52:52 +0530 Subject: [PATCH 7/9] add log for blacklist --- code/go/0chain.net/blobbercore/handler/client_quota.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/code/go/0chain.net/blobbercore/handler/client_quota.go b/code/go/0chain.net/blobbercore/handler/client_quota.go index cee45a830..4335192d4 100644 --- a/code/go/0chain.net/blobbercore/handler/client_quota.go +++ b/code/go/0chain.net/blobbercore/handler/client_quota.go @@ -8,6 +8,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" "gorm.io/gorm" ) @@ -84,6 +86,7 @@ func AddWriteMarkerCount(clientID string, zeroSizeWM bool) { cs.TotalZeroWM++ } if cs.TotalZeroWM > config.Configuration.CommitZeroLimitDaily || cs.TotalWM > config.Configuration.CommitLimitDaily { + logging.Logger.Info("Client blacklisted", zap.String("client_id", clientID), zap.Int64("total_write_marker", cs.TotalWM), zap.Int64("total_zero_write_marker", cs.TotalZeroWM), zap.Int64("commit_limit_daily", config.Configuration.CommitLimitDaily), zap.Int64("commit_zero_limit_daily", config.Configuration.CommitZeroLimitDaily)) SetBlacklist(clientID) } } @@ -136,6 +139,7 @@ func saveClientStats() { } func startBlackListWorker(ctx context.Context) { + logging.Logger.Info("Starting black list worker", zap.Int64("upload_limit", config.Configuration.UploadLimitMonthly), zap.Int64("download_limit", config.Configuration.BlockLimitMonthly), zap.Int64("commit_limit", config.Configuration.CommitLimitMonthly), zap.Int64("commit_zero_limit", config.Configuration.CommitZeroLimitDaily), zap.Int64("commit_limit_daily", config.Configuration.CommitLimitDaily)) BlackListWorkerTime := 24 * time.Hour if config.Development() { BlackListWorkerTime = 10 * time.Second From dfd6a266b6985f79e3d6a927aac337b48e255e27 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 30 Sep 2024 20:53:43 +0530 Subject: [PATCH 8/9] add get refs log --- code/go/0chain.net/blobbercore/reference/referencepath.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 8972d50ec..d044b6465 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -247,7 +247,9 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, path = filepath.Clean(path) tx := datastore.GetStore().GetTransaction(ctx) pathLevel := len(strings.Split(strings.TrimSuffix(path, "/"), "/")) + logging.Logger.Info("getRefs: CheckSingleRef", zap.Int("pathLevel", pathLevel), zap.Int("level", level), zap.String("path", path), zap.String("offsetPath", offsetPath), zap.String("type", _type), zap.Int("pageLimit", pageLimit)) if (pageLimit == 1 && offsetPath == "" && (pathLevel == level || level == 0) && _type != FILE) || (parentRef != nil && parentRef.Type == FILE) { + logging.Logger.Info("getRefs: SingleRef") pRefs = append(pRefs, *parentRef) refs = &pRefs newOffsetPath = parentRef.Path @@ -268,6 +270,7 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, if path != "/" { listPath = path + "/" } + logging.Logger.Info("getRefs: ListRef", zap.String("listPath", listPath)) dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND path LIKE ?", allocationID, listPath+"%") if _type != "" { dbQuery = dbQuery.Where("type = ?", _type) From 991b6388665e8c70de320b0ab4ec934ed54674bb Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 30 Sep 2024 21:31:49 +0530 Subject: [PATCH 9/9] add return --- code/go/0chain.net/blobbercore/handler/storage_handler.go | 3 +++ code/go/0chain.net/blobbercore/reference/referencepath.go | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 2e880ec9f..56625bf28 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -943,6 +943,9 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb refs, totalPages, newOffsetPath, err = reference.GetRefs( ctx, allocationID, path, offsetPath, fileType, level, pageLimit, offsetTime, pathRef, ) + if refs != nil { + logging.Logger.Info("GetRefs: regular", zap.Int("refs", len(*refs)), zap.Int("totalPages", totalPages), zap.String("newOffsetPath", newOffsetPath), zap.Error(err)) + } case refType == "updated": refs, totalPages, newOffsetPath, newOffsetDate, err = reference.GetUpdatedRefs( diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index d044b6465..f2bfc7830 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -249,10 +249,10 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, pathLevel := len(strings.Split(strings.TrimSuffix(path, "/"), "/")) logging.Logger.Info("getRefs: CheckSingleRef", zap.Int("pathLevel", pathLevel), zap.Int("level", level), zap.String("path", path), zap.String("offsetPath", offsetPath), zap.String("type", _type), zap.Int("pageLimit", pageLimit)) if (pageLimit == 1 && offsetPath == "" && (pathLevel == level || level == 0) && _type != FILE) || (parentRef != nil && parentRef.Type == FILE) { - logging.Logger.Info("getRefs: SingleRef") pRefs = append(pRefs, *parentRef) refs = &pRefs newOffsetPath = parentRef.Path + return } if pathLevel+1 == level { @@ -270,7 +270,6 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, if path != "/" { listPath = path + "/" } - logging.Logger.Info("getRefs: ListRef", zap.String("listPath", listPath)) dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND path LIKE ?", allocationID, listPath+"%") if _type != "" { dbQuery = dbQuery.Where("type = ?", _type)