Download block limit (#1254)

* add limit as 500 to numBlocks

* add daily and req limit

* fix test

* fix test

code/go/0chain.net/blobber/config.go

@@ -97,6 +97,9 @@ func setupConfig(configDir string, deploymentMode int) {
 		config.Configuration.MinConfirmation = 100
 	}
 
+	config.Configuration.BlockLimitDaily = viper.GetInt64("rate_limiters.block_limit_daily")
+	config.Configuration.BlockLimitRequest = viper.GetInt64("rate_limiters.block_limit_request")
+
 	transaction.MinConfirmation = config.Configuration.MinConfirmation
 
 	fmt.Print("		[OK]\n")

code/go/0chain.net/blobbercore/config/config.go

@@ -22,6 +22,8 @@ func SetupDefaultConfig() {
 	viper.SetDefault("challenge_response.frequency", 10)
 	viper.SetDefault("challenge_response.num_workers", 5)
 	viper.SetDefault("challenge_response.max_retries", 10)
+	viper.SetDefault("rate_limiters.block_limit_daily", 1562500)
+	viper.SetDefault("rate_limiters.block_limit_request", 500)
 
 	viper.SetDefault("healthcheck.frequency", "60s")
 
@@ -87,6 +89,8 @@ type Config struct {
 	ChallengeMaxRetires           int
 	TempFilesCleanupFreq          int64
 	TempFilesCleanupNumWorkers    int
+	BlockLimitDaily               int64
+	BlockLimitRequest             int64
 
 	HealthCheckWorkerFreq time.Duration
 

code/go/0chain.net/blobbercore/handler/download_quota.go

@@ -1,8 +1,10 @@
 package handler
 
 import (
+	"context"
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/0chain/blobber/code/go/0chain.net/core/common"
 )
@@ -17,8 +19,37 @@ type QuotaManager struct {
 	mux sync.RWMutex
 }
 
-var quotaManagerInstance *QuotaManager
-var quotaManagerOnce sync.Once
+var (
+	quotaManagerInstance *QuotaManager
+	quotaManagerOnce     sync.Once
+	downloadLimit        = make(map[string]int64)
+	downloadLock         sync.RWMutex
+)
+
+func addDailyBlocks(key string, numBlocks int64) {
+	downloadLock.Lock()
+	defer downloadLock.Unlock()
+	downloadLimit[key] += numBlocks
+}
+
+func getDailyBlocks(key string) int64 {
+	downloadLock.RLock()
+	defer downloadLock.RUnlock()
+	return downloadLimit[key]
+}
+
+func startDownloadLimitCleanup(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(24 * time.Hour):
+			downloadLock.Lock()
+			downloadLimit = make(map[string]int64)
+			downloadLock.Unlock()
+		}
+	}
+}
 
 func getQuotaManager() *QuotaManager {
 	quotaManagerOnce.Do(func() {

code/go/0chain.net/blobbercore/handler/handler_test.go

@@ -28,6 +28,7 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation"
+	blobConfig "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
 	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore"
 	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore"
 	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference"
@@ -72,6 +73,8 @@ func init() {
 	resetMockFileBlock()
 	chain.SetServerChain(&chain.Chain{})
 	config.Configuration.SignatureScheme = "bls0chain"
+	blobConfig.Configuration.BlockLimitDaily = 1562500
+	blobConfig.Configuration.BlockLimitRequest = 500
 	logging.Logger = zap.NewNop()
 	ConfigRateLimits()
 

code/go/0chain.net/blobbercore/handler/object_operation_handler.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp"
+	"github.com/0chain/blobber/code/go/0chain.net/blobbercore/config"
 
 	"github.com/0chain/gosdk/constants"
 
@@ -287,6 +288,15 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i
 		return nil, err
 	}
 
+	if dr.NumBlocks > config.Configuration.BlockLimitRequest {
+		return nil, common.NewErrorf("download_file", "too many blocks requested: %v, max limit is %v", dr.NumBlocks, config.Configuration.BlockLimitRequest)
+	}
+
+	dailyBlocksConsumed := getDailyBlocks(clientID)
+	if dailyBlocksConsumed+dr.NumBlocks > config.Configuration.BlockLimitDaily {
+		return nil, common.NewErrorf("download_file", "daily block limit reached: %v, max limit is %v", dailyBlocksConsumed, config.Configuration.BlockLimitDaily)
+	}
+
 	fileref, err := reference.GetReferenceByLookupHash(ctx, alloc.ID, dr.PathHash)
 	if err != nil {
 		return nil, common.NewErrorf("download_file", "invalid file path: %v", err)
@@ -416,6 +426,7 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i
 
 	fileDownloadResponse.Data = chunkData
 	reference.FileBlockDownloaded(ctx, fileref.ID, dr.NumBlocks)
+	addDailyBlocks(clientID, dr.NumBlocks)
 	return fileDownloadResponse, nil
 }
 

code/go/0chain.net/blobbercore/handler/worker.go

@@ -17,6 +17,7 @@ import (
 
 func SetupWorkers(ctx context.Context) {
 	go startCleanupTempFiles(ctx)
+	go startDownloadLimitCleanup(ctx)
 }
 
 func CleanupDiskFiles(ctx context.Context) error {

config/0chain_blobber.yaml

@@ -59,7 +59,10 @@ rate_limiters:
   # General Request Per Second. This rps is used to rate limit endpoints like copy, rename, get file metadata,
   # get paginated refs, etc. Default is 5
   general_rps: 1600
-
+  # Number of blocks downloaded in a day. Default is 100GB(the value needs to be in blocks which is data/64KB)
+  block_limit_daily: 1562500
+  # Max blocks per download request. Default is 500
+  block_limit_request: 500  
 server_chain:
   id: "0afc093ffb509f059c55478bc1a60351cef7b4e9c008a53a6cc8241ca8617dfe"
   owner: "edb90b850f2e7e7cbd0a1fa370fdcc5cd378ffbec95363a7bc0e5a98b8ba5759"