diff --git a/code/go/0chain.net/blobber/config.go b/code/go/0chain.net/blobber/config.go index 55a80b794..a8f30bfa3 100644 --- a/code/go/0chain.net/blobber/config.go +++ b/code/go/0chain.net/blobber/config.go @@ -97,6 +97,9 @@ func setupConfig(configDir string, deploymentMode int) { config.Configuration.MinConfirmation = 100 } + config.Configuration.BlockLimitDaily = viper.GetInt64("rate_limiters.block_limit_daily") + config.Configuration.BlockLimitRequest = viper.GetInt64("rate_limiters.block_limit_request") + transaction.MinConfirmation = config.Configuration.MinConfirmation fmt.Print(" [OK]\n") diff --git a/code/go/0chain.net/blobbercore/config/config.go b/code/go/0chain.net/blobbercore/config/config.go index 06170e871..db7e359c9 100644 --- a/code/go/0chain.net/blobbercore/config/config.go +++ b/code/go/0chain.net/blobbercore/config/config.go @@ -22,6 +22,8 @@ func SetupDefaultConfig() { viper.SetDefault("challenge_response.frequency", 10) viper.SetDefault("challenge_response.num_workers", 5) viper.SetDefault("challenge_response.max_retries", 10) + viper.SetDefault("rate_limiters.block_limit_daily", 1562500) + viper.SetDefault("rate_limiters.block_limit_request", 500) viper.SetDefault("healthcheck.frequency", "60s") @@ -87,6 +89,8 @@ type Config struct { ChallengeMaxRetires int TempFilesCleanupFreq int64 TempFilesCleanupNumWorkers int + BlockLimitDaily int64 + BlockLimitRequest int64 HealthCheckWorkerFreq time.Duration diff --git a/code/go/0chain.net/blobbercore/handler/download_quota.go b/code/go/0chain.net/blobbercore/handler/download_quota.go index 8d731cc5f..4f14b4197 100644 --- a/code/go/0chain.net/blobbercore/handler/download_quota.go +++ b/code/go/0chain.net/blobbercore/handler/download_quota.go @@ -1,8 +1,10 @@ package handler import ( + "context" "fmt" "sync" + "time" "github.com/0chain/blobber/code/go/0chain.net/core/common" ) @@ -17,8 +19,37 @@ type QuotaManager struct { mux sync.RWMutex } -var quotaManagerInstance *QuotaManager -var quotaManagerOnce sync.Once +var ( + quotaManagerInstance *QuotaManager + quotaManagerOnce sync.Once + downloadLimit = make(map[string]int64) + downloadLock sync.RWMutex +) + +func addDailyBlocks(key string, numBlocks int64) { + downloadLock.Lock() + defer downloadLock.Unlock() + downloadLimit[key] += numBlocks +} + +func getDailyBlocks(key string) int64 { + downloadLock.RLock() + defer downloadLock.RUnlock() + return downloadLimit[key] +} + +func startDownloadLimitCleanup(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case <-time.After(24 * time.Hour): + downloadLock.Lock() + downloadLimit = make(map[string]int64) + downloadLock.Unlock() + } + } +} func getQuotaManager() *QuotaManager { quotaManagerOnce.Do(func() { diff --git a/code/go/0chain.net/blobbercore/handler/handler_test.go b/code/go/0chain.net/blobbercore/handler/handler_test.go index 49a7b712a..4f4b375a5 100644 --- a/code/go/0chain.net/blobbercore/handler/handler_test.go +++ b/code/go/0chain.net/blobbercore/handler/handler_test.go @@ -28,6 +28,7 @@ import ( "go.uber.org/zap" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/allocation" + blobConfig "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/filestore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/reference" @@ -72,6 +73,8 @@ func init() { resetMockFileBlock() chain.SetServerChain(&chain.Chain{}) config.Configuration.SignatureScheme = "bls0chain" + blobConfig.Configuration.BlockLimitDaily = 1562500 + blobConfig.Configuration.BlockLimitRequest = 500 logging.Logger = zap.NewNop() ConfigRateLimits() diff --git a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go index a9ae2c411..ae89a1097 100644 --- a/code/go/0chain.net/blobbercore/handler/object_operation_handler.go +++ b/code/go/0chain.net/blobbercore/handler/object_operation_handler.go @@ -13,6 +13,7 @@ import ( "time" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/blobberhttp" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/config" "github.com/0chain/gosdk/constants" @@ -287,6 +288,15 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i return nil, err } + if dr.NumBlocks > config.Configuration.BlockLimitRequest { + return nil, common.NewErrorf("download_file", "too many blocks requested: %v, max limit is %v", dr.NumBlocks, config.Configuration.BlockLimitRequest) + } + + dailyBlocksConsumed := getDailyBlocks(clientID) + if dailyBlocksConsumed+dr.NumBlocks > config.Configuration.BlockLimitDaily { + return nil, common.NewErrorf("download_file", "daily block limit reached: %v, max limit is %v", dailyBlocksConsumed, config.Configuration.BlockLimitDaily) + } + fileref, err := reference.GetReferenceByLookupHash(ctx, alloc.ID, dr.PathHash) if err != nil { return nil, common.NewErrorf("download_file", "invalid file path: %v", err) @@ -416,6 +426,7 @@ func (fsh *StorageHandler) DownloadFile(ctx context.Context, r *http.Request) (i fileDownloadResponse.Data = chunkData reference.FileBlockDownloaded(ctx, fileref.ID, dr.NumBlocks) + addDailyBlocks(clientID, dr.NumBlocks) return fileDownloadResponse, nil } diff --git a/code/go/0chain.net/blobbercore/handler/worker.go b/code/go/0chain.net/blobbercore/handler/worker.go index 2634b148e..56eefb7bf 100644 --- a/code/go/0chain.net/blobbercore/handler/worker.go +++ b/code/go/0chain.net/blobbercore/handler/worker.go @@ -17,6 +17,7 @@ import ( func SetupWorkers(ctx context.Context) { go startCleanupTempFiles(ctx) + go startDownloadLimitCleanup(ctx) } func CleanupDiskFiles(ctx context.Context) error { diff --git a/config/0chain_blobber.yaml b/config/0chain_blobber.yaml index b4c845b25..be1122ef6 100755 --- a/config/0chain_blobber.yaml +++ b/config/0chain_blobber.yaml @@ -59,7 +59,10 @@ rate_limiters: # General Request Per Second. This rps is used to rate limit endpoints like copy, rename, get file metadata, # get paginated refs, etc. Default is 5 general_rps: 1600 - + # Number of blocks downloaded in a day. Default is 100GB(the value needs to be in blocks which is data/64KB) + block_limit_daily: 1562500 + # Max blocks per download request. Default is 500 + block_limit_request: 500 server_chain: id: "0afc093ffb509f059c55478bc1a60351cef7b4e9c008a53a6cc8241ca8617dfe" owner: "edb90b850f2e7e7cbd0a1fa370fdcc5cd378ffbec95363a7bc0e5a98b8ba5759"