diff --git a/.github/workflows/build-&-publish-docker-image.yml b/.github/workflows/build-&-publish-docker-image.yml index dbd46074d..9bfb20263 100644 --- a/.github/workflows/build-&-publish-docker-image.yml +++ b/.github/workflows/build-&-publish-docker-image.yml @@ -47,6 +47,12 @@ jobs: # with: # go-version: ^1.21 # The Go version to download (if necessary) and use. + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Clone blobber uses: actions/checkout@v3 with: @@ -56,10 +62,9 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} + run: | + mkdir -p /tmp/docker-config-${{ github.run_id }} + docker --config /tmp/docker-config-${{ github.run_id }} login -u ${{ secrets.DOCKERHUB_USERNAME }} -p ${{ secrets.DOCKERHUB_PASSWORD }} # - name: Get changed files using defaults # id: changed-files @@ -96,6 +101,10 @@ jobs: docker tag ${BLOBBER_REGISTRY}:${TAG} ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA} docker push ${BLOBBER_REGISTRY}:${TAG}-${SHORT_SHA} + - name: Clean up Docker Config + run: | + rm -rf /tmp/docker-config-${{ github.run_id }} + validator: timeout-minutes: 30 runs-on: [blobber-runner] @@ -136,11 +145,16 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} + run: | + mkdir -p /tmp/docker-config-${{ github.run_id }} + docker --config /tmp/docker-config-${{ github.run_id }} login -u ${{ secrets.DOCKERHUB_USERNAME }} -p ${{ secrets.DOCKERHUB_PASSWORD }} + # uses: docker/login-action@v1 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_PASSWORD }} + # - name: Get changed files using defaults # id: changed-files @@ -178,6 +192,10 @@ jobs: docker tag ${VALIDATOR_REGISTRY}:${TAG} ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA} docker push ${VALIDATOR_REGISTRY}:${TAG}-${SHORT_SHA} + - name: Clean up Docker Config + run: | + rm -rf /tmp/docker-config-${{ github.run_id }} + system-tests: if: github.event_name != 'workflow_dispatch' needs: [blobber, validator] diff --git a/code/go/0chain.net/blobbercore/allocation/connection.go b/code/go/0chain.net/blobbercore/allocation/connection.go index 75ac30b0c..47e418712 100644 --- a/code/go/0chain.net/blobbercore/allocation/connection.go +++ b/code/go/0chain.net/blobbercore/allocation/connection.go @@ -246,6 +246,7 @@ func GetHasher(connectionID, pathHash string) *filestore.CommitHasher { // DeleteConnectionObjEntry remove the connectionID entry from map // If the given connectionID is not present, then it is no-op. func DeleteConnectionObjEntry(connectionID string) { + logging.Logger.Info("DeleteConnectionObjEntry", zap.String("connection_id", connectionID)) connectionObjMutex.Lock() connectionObj, ok := connectionProcessor[connectionID] if ok { @@ -262,6 +263,7 @@ func cleanConnectionObj() { for connectionID, connectionObj := range connectionProcessor { diff := time.Since(connectionObj.UpdatedAt) if diff >= ConnectionObjTimeout { + logging.Logger.Info("cleanConnectionObj", zap.String("connection_id", connectionID), zap.Duration("diff", diff)) // Stop the context and hash worker connectionObj.cnclCtx() for _, change := range connectionObj.changes { diff --git a/code/go/0chain.net/blobbercore/handler/client_quota.go b/code/go/0chain.net/blobbercore/handler/client_quota.go index cee45a830..4335192d4 100644 --- a/code/go/0chain.net/blobbercore/handler/client_quota.go +++ b/code/go/0chain.net/blobbercore/handler/client_quota.go @@ -8,6 +8,8 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" + "github.com/0chain/blobber/code/go/0chain.net/core/logging" + "go.uber.org/zap" "gorm.io/gorm" ) @@ -84,6 +86,7 @@ func AddWriteMarkerCount(clientID string, zeroSizeWM bool) { cs.TotalZeroWM++ } if cs.TotalZeroWM > config.Configuration.CommitZeroLimitDaily || cs.TotalWM > config.Configuration.CommitLimitDaily { + logging.Logger.Info("Client blacklisted", zap.String("client_id", clientID), zap.Int64("total_write_marker", cs.TotalWM), zap.Int64("total_zero_write_marker", cs.TotalZeroWM), zap.Int64("commit_limit_daily", config.Configuration.CommitLimitDaily), zap.Int64("commit_zero_limit_daily", config.Configuration.CommitZeroLimitDaily)) SetBlacklist(clientID) } } @@ -136,6 +139,7 @@ func saveClientStats() { } func startBlackListWorker(ctx context.Context) { + logging.Logger.Info("Starting black list worker", zap.Int64("upload_limit", config.Configuration.UploadLimitMonthly), zap.Int64("download_limit", config.Configuration.BlockLimitMonthly), zap.Int64("commit_limit", config.Configuration.CommitLimitMonthly), zap.Int64("commit_zero_limit", config.Configuration.CommitZeroLimitDaily), zap.Int64("commit_limit_daily", config.Configuration.CommitLimitDaily)) BlackListWorkerTime := 24 * time.Hour if config.Development() { BlackListWorkerTime = 10 * time.Second diff --git a/code/go/0chain.net/blobbercore/handler/storage_handler.go b/code/go/0chain.net/blobbercore/handler/storage_handler.go index 2e880ec9f..56625bf28 100644 --- a/code/go/0chain.net/blobbercore/handler/storage_handler.go +++ b/code/go/0chain.net/blobbercore/handler/storage_handler.go @@ -943,6 +943,9 @@ func (fsh *StorageHandler) GetRefs(ctx context.Context, r *http.Request) (*blobb refs, totalPages, newOffsetPath, err = reference.GetRefs( ctx, allocationID, path, offsetPath, fileType, level, pageLimit, offsetTime, pathRef, ) + if refs != nil { + logging.Logger.Info("GetRefs: regular", zap.Int("refs", len(*refs)), zap.Int("totalPages", totalPages), zap.String("newOffsetPath", newOffsetPath), zap.Error(err)) + } case refType == "updated": refs, totalPages, newOffsetPath, newOffsetDate, err = reference.GetUpdatedRefs( diff --git a/code/go/0chain.net/blobbercore/handler/worker.go b/code/go/0chain.net/blobbercore/handler/worker.go index 19cf49c30..b5acbe174 100644 --- a/code/go/0chain.net/blobbercore/handler/worker.go +++ b/code/go/0chain.net/blobbercore/handler/worker.go @@ -87,6 +87,10 @@ func cleanupTempFiles(ctx context.Context) { for i := 0; i < len(openConnectionsToDelete); i++ { connection := &openConnectionsToDelete[i] logging.Logger.Info("Deleting temp files for the connection", zap.Any("connection", connection.ID)) + processor := allocation.GetConnectionProcessor(connection.ID) + if processor != nil { + continue + } connection.ComputeProperties() nctx := datastore.GetStore().CreateTransaction(ctx) diff --git a/code/go/0chain.net/blobbercore/reference/referencepath.go b/code/go/0chain.net/blobbercore/reference/referencepath.go index 16b0e6884..f2bfc7830 100644 --- a/code/go/0chain.net/blobbercore/reference/referencepath.go +++ b/code/go/0chain.net/blobbercore/reference/referencepath.go @@ -247,10 +247,12 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, path = filepath.Clean(path) tx := datastore.GetStore().GetTransaction(ctx) pathLevel := len(strings.Split(strings.TrimSuffix(path, "/"), "/")) + logging.Logger.Info("getRefs: CheckSingleRef", zap.Int("pathLevel", pathLevel), zap.Int("level", level), zap.String("path", path), zap.String("offsetPath", offsetPath), zap.String("type", _type), zap.Int("pageLimit", pageLimit)) if (pageLimit == 1 && offsetPath == "" && (pathLevel == level || level == 0) && _type != FILE) || (parentRef != nil && parentRef.Type == FILE) { pRefs = append(pRefs, *parentRef) refs = &pRefs newOffsetPath = parentRef.Path + return } if pathLevel+1 == level { @@ -264,7 +266,11 @@ func GetRefs(ctx context.Context, allocationID, path, offsetPath, _type string, } dbQuery = dbQuery.Order("path") } else { - dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND path LIKE ?", allocationID, path+"%") + listPath := path + if path != "/" { + listPath = path + "/" + } + dbQuery = tx.Model(&Ref{}).Where("allocation_id = ? AND path LIKE ?", allocationID, listPath+"%") if _type != "" { dbQuery = dbQuery.Where("type = ?", _type) } diff --git a/docker.local/blobber.Dockerfile b/docker.local/blobber.Dockerfile index fdfb40ee8..c66835945 100644 --- a/docker.local/blobber.Dockerfile +++ b/docker.local/blobber.Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 ARG DOCKER_IMAGE_BASE -FROM $DOCKER_IMAGE_BASE as blobber_build +FROM $DOCKER_IMAGE_BASE AS blobber_build LABEL zchain="blobber" ENV SRC_DIR=/0chain diff --git a/docker.local/validator.Dockerfile b/docker.local/validator.Dockerfile index 7c1313cc6..0ae0c0628 100644 --- a/docker.local/validator.Dockerfile +++ b/docker.local/validator.Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 ARG DOCKER_IMAGE_BASE -FROM $DOCKER_IMAGE_BASE as validator_build +FROM $DOCKER_IMAGE_BASE AS validator_build LABEL zchain="validator"