From 50a4984be75bb434b60ef33a9714d506e35fec5a Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 23 Oct 2023 10:58:50 +0530 Subject: [PATCH 1/4] fix blobber stats --- code/go/0chain.net/blobbercore/handler/handler.go | 4 +--- code/go/0chain.net/blobbercore/stats/blobberstats.go | 12 +++--------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index f658d2448..dc179833c 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -226,8 +226,6 @@ func setupHandlers(r *mux.Router) { r.HandleFunc("/_logs", RateLimitByCommmitRL(common.ToJSONResponse(GetLogs))) - // r.HandleFunc("/_statsJSON", common.AuthenticateAdmin(common.ToJSONResponse(stats.StatsJSONHandler))) - r.HandleFunc("/_statsJSON", RateLimitByCommmitRL(common.ToJSONResponse(stats.StatsJSONHandler))) // r.HandleFunc("/_cleanupdisk", common.AuthenticateAdmin(common.ToJSONResponse(WithReadOnlyConnection(CleanupDiskHandler)))) // r.HandleFunc("/_cleanupdisk", RateLimitByCommmitRL(common.ToJSONResponse(WithReadOnlyConnection(CleanupDiskHandler)))) // r.HandleFunc("/getstats", common.AuthenticateAdmin(common.ToJSONResponse(stats.GetStatsHandler))) @@ -745,7 +743,7 @@ func StatsHandler(w http.ResponseWriter, r *http.Request) { writeResponse(w, []byte(err.Error())) return } - + w.Header().Set("Content-Type", "application/json") writeResponse(w, statsJson) return diff --git a/code/go/0chain.net/blobbercore/stats/blobberstats.go b/code/go/0chain.net/blobbercore/stats/blobberstats.go index 3ae93454b..95d3bf3bc 100644 --- a/code/go/0chain.net/blobbercore/stats/blobberstats.go +++ b/code/go/0chain.net/blobbercore/stats/blobberstats.go @@ -211,14 +211,10 @@ func (bs *BlobberStats) loadStats(ctx context.Context) { const sel = ` COALESCE (SUM (reference_objects.size), 0) AS files_size, COALESCE (SUM (reference_objects.thumbnail_size), 0) AS thumbnails_size, - COALESCE (SUM (file_stats.num_of_block_downloads), 0) AS num_of_reads, + COALESCE (SUM (reference_objects.num_of_block_downloads), 0) AS num_of_reads, COALESCE (SUM (reference_objects.num_of_blocks), 0) AS num_of_block_writes, COUNT (*) AS num_of_writes` - const join = ` - INNER JOIN file_stats ON reference_objects.id = file_stats.ref_id - WHERE reference_objects.type = 'f'` - var ( db = datastore.GetStore().GetTransaction(ctx) row *sql.Row @@ -227,7 +223,7 @@ func (bs *BlobberStats) loadStats(ctx context.Context) { row = db.Table("reference_objects"). Select(sel). - Joins(join). + Where("reference_objects.type = 'f'"). Row() err = row.Scan(&bs.FilesSize, &bs.ThumbnailsSize, &bs.NumReads, @@ -269,13 +265,11 @@ func (bs *BlobberStats) loadAllocationStats(ctx context.Context) { reference_objects.allocation_id, SUM(reference_objects.size) as files_size, SUM(reference_objects.thumbnail_size) as thumbnails_size, - SUM(file_stats.num_of_block_downloads) as num_of_reads, + SUM(reference_objects.num_of_block_downloads) as num_of_reads, SUM(reference_objects.num_of_blocks) as num_of_block_writes, COUNT(*) as num_of_writes, allocations.blobber_size AS allocated_size, allocations.expiration_date AS expiration_date`). - Joins(`INNER JOIN file_stats - ON reference_objects.id = file_stats.ref_id`). Joins(` INNER JOIN allocations ON allocations.id = reference_objects.allocation_id`). From fdb90d9e4f9c76a47295685634ffe4a99d6f56f6 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Tue, 24 Oct 2023 00:38:45 +0530 Subject: [PATCH 2/4] add stats worker --- code/go/0chain.net/blobber/http.go | 8 +++--- code/go/0chain.net/blobber/worker.go | 2 ++ .../0chain.net/blobbercore/handler/handler.go | 14 +++++----- .../blobbercore/stats/blobberstats.go | 27 ++++++++++++++++--- .../0chain.net/blobbercore/stats/handler.go | 24 ----------------- code/go/0chain.net/core/common/admin.go | 26 ++++++++++-------- 6 files changed, 50 insertions(+), 51 deletions(-) diff --git a/code/go/0chain.net/blobber/http.go b/code/go/0chain.net/blobber/http.go index 0ec89c658..9846c27f4 100644 --- a/code/go/0chain.net/blobber/http.go +++ b/code/go/0chain.net/blobber/http.go @@ -26,7 +26,7 @@ func startHttpServer() { } r := mux.NewRouter() - initHandlers(r) + initHandlers(r, config.Development()) var wg sync.WaitGroup @@ -36,7 +36,7 @@ func startHttpServer() { // start https server go startServer(&wg, r, mode, httpsPort, true) - logging.Logger.Info("Ready to listen to the requests") + logging.Logger.Info("Ready to listen to the requests with development mode: " + mode) fmt.Print("> start http server [OK]\n") wg.Wait() @@ -104,12 +104,12 @@ func startServer(wg *sync.WaitGroup, r *mux.Router, mode string, port int, isTls } } -func initHandlers(r *mux.Router) { +func initHandlers(r *mux.Router, devMode bool) { handler.StartTime = time.Now().UTC() r.HandleFunc("/", handler.HomepageHandler) handler.SetupHandlers(r) handler.SetupSwagger() - common.SetAdminCredentials() + common.SetAdminCredentials(devMode) } func initProfHandlers(mux *http.ServeMux) { diff --git a/code/go/0chain.net/blobber/worker.go b/code/go/0chain.net/blobber/worker.go index dc2dae658..f42bbc76e 100644 --- a/code/go/0chain.net/blobber/worker.go +++ b/code/go/0chain.net/blobber/worker.go @@ -10,6 +10,7 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/readmarker" + "github.com/0chain/blobber/code/go/0chain.net/blobbercore/stats" "github.com/0chain/blobber/code/go/0chain.net/blobbercore/writemarker" "github.com/0chain/blobber/code/go/0chain.net/core/logging" @@ -26,6 +27,7 @@ func setupWorkers(ctx context.Context) { allocation.SetupWorkers(ctx) challenge.SetupChallengeCleanUpWorker(ctx) challenge.SetupChallengeTimingsCleanupWorker(ctx) + stats.SetupStatsWorker(ctx) updateCCTWorker(ctx) } diff --git a/code/go/0chain.net/blobbercore/handler/handler.go b/code/go/0chain.net/blobbercore/handler/handler.go index dc179833c..a16887b2a 100644 --- a/code/go/0chain.net/blobbercore/handler/handler.go +++ b/code/go/0chain.net/blobbercore/handler/handler.go @@ -217,10 +217,10 @@ func setupHandlers(r *mux.Router) { // Allowing admin api for debugging purpose only. Later on commented out line should be // uncommented and line below it should be deleted - // r.HandleFunc("/_debug", common.AuthenticateAdmin(common.ToJSONResponse(DumpGoRoutines))) - r.HandleFunc("/_debug", RateLimitByCommmitRL(common.ToJSONResponse(DumpGoRoutines))) - // r.HandleFunc("/_config", common.AuthenticateAdmin(common.ToJSONResponse(GetConfig))) - r.HandleFunc("/_config", RateLimitByCommmitRL(common.ToJSONResponse(GetConfig))) + r.HandleFunc("/_debug", common.AuthenticateAdmin(common.ToJSONResponse(DumpGoRoutines))) + // r.HandleFunc("/_debug", RateLimitByCommmitRL(common.ToJSONResponse(DumpGoRoutines))) + r.HandleFunc("/_config", common.AuthenticateAdmin(common.ToJSONResponse(GetConfig))) + // r.HandleFunc("/_config", RateLimitByCommmitRL(common.ToJSONResponse(GetConfig))) // r.HandleFunc("/_stats", common.AuthenticateAdmin(StatsHandler)) r.HandleFunc("/_stats", RateLimitByCommmitRL(StatsHandler)) @@ -228,10 +228,8 @@ func setupHandlers(r *mux.Router) { // r.HandleFunc("/_cleanupdisk", common.AuthenticateAdmin(common.ToJSONResponse(WithReadOnlyConnection(CleanupDiskHandler)))) // r.HandleFunc("/_cleanupdisk", RateLimitByCommmitRL(common.ToJSONResponse(WithReadOnlyConnection(CleanupDiskHandler)))) - // r.HandleFunc("/getstats", common.AuthenticateAdmin(common.ToJSONResponse(stats.GetStatsHandler))) - r.HandleFunc("/getstats", RateLimitByCommmitRL(common.ToJSONResponse(WithReadOnlyConnection(stats.GetStatsHandler)))) - // r.HandleFunc("/challengetimings", common.AuthenticateAdmin(common.ToJSONResponse(GetChallengeTimings))) - r.HandleFunc("/challengetimings", RateLimitByCommmitRL(common.ToJSONResponse(GetChallengeTimings))) + r.HandleFunc("/challengetimings", common.AuthenticateAdmin(common.ToJSONResponse(GetChallengeTimings))) + // r.HandleFunc("/challengetimings", RateLimitByCommmitRL(common.ToJSONResponse(GetChallengeTimings))) r.HandleFunc("/challenge-timings-by-challengeId", RateLimitByCommmitRL(common.ToJSONResponse(GetChallengeTiming))) //marketplace related diff --git a/code/go/0chain.net/blobbercore/stats/blobberstats.go b/code/go/0chain.net/blobbercore/stats/blobberstats.go index 95d3bf3bc..f4e0f1352 100644 --- a/code/go/0chain.net/blobbercore/stats/blobberstats.go +++ b/code/go/0chain.net/blobbercore/stats/blobberstats.go @@ -83,17 +83,36 @@ type BlobberStats struct { WriteMarkers WriteMarkersStat `json:"write_markers"` } +var fs *BlobberStats + +const statsHandlerPeriod = 30 * time.Minute + type AllocationId struct { Id string `json:"id"` } +func SetupStatsWorker(ctx context.Context) { + fs = &BlobberStats{} + go func() { + for { + select { + case <-ctx.Done(): + return + case <-time.After(statsHandlerPeriod): + _ = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + fs.loadBasicStats(ctx) + fs.loadDetailedStats(ctx) + fs.loadFailedChallengeList(ctx) + return common.NewError("rollback", "read_only") + }) + } + } + }() +} + func LoadBlobberStats(ctx context.Context) *BlobberStats { - fs := &BlobberStats{} - fs.loadBasicStats(ctx) - fs.loadDetailedStats(ctx) fs.loadInfraStats(ctx) fs.loadDBStats() - fs.loadFailedChallengeList(ctx) return fs } diff --git a/code/go/0chain.net/blobbercore/stats/handler.go b/code/go/0chain.net/blobbercore/stats/handler.go index c3821ec5b..e6d7e7eea 100644 --- a/code/go/0chain.net/blobbercore/stats/handler.go +++ b/code/go/0chain.net/blobbercore/stats/handler.go @@ -10,7 +10,6 @@ import ( "github.com/0chain/blobber/code/go/0chain.net/blobbercore/datastore" "github.com/0chain/blobber/code/go/0chain.net/core/common" . "github.com/0chain/blobber/code/go/0chain.net/core/logging" - "github.com/0chain/gosdk/constants" "go.uber.org/zap" ) @@ -485,26 +484,3 @@ func StatsJSONHandler(ctx context.Context, r *http.Request) (interface{}, error) bs := LoadBlobberStats(ctx) return bs, nil } - -func GetStatsHandler(ctx context.Context, r *http.Request) (interface{}, error) { - q := r.URL.Query() - ctx = context.WithValue(ctx, constants.ContextKeyAllocation, q.Get("allocation_id")) - allocationID := ctx.Value(constants.ContextKeyAllocation).(string) - bs := &BlobberStats{} - if allocationID != "" { - // TODO: Get only the allocation info from DB - bs.loadDetailedStats(ctx) - for _, allocStat := range bs.AllocationStats { - if allocStat.AllocationID == allocationID { - return allocStat, nil - } - } - return nil, common.NewError("allocation_stats_not_found", "Stats for allocation not found") - } - allocations := q.Get("allocations") - if allocations != "" { - return loadAllocationList(ctx) - } - bs.loadBasicStats(ctx) - return bs, nil -} diff --git a/code/go/0chain.net/core/common/admin.go b/code/go/0chain.net/core/common/admin.go index 5cd87800e..d43a75fbf 100644 --- a/code/go/0chain.net/core/common/admin.go +++ b/code/go/0chain.net/core/common/admin.go @@ -8,25 +8,29 @@ import ( // global username and password used to access endpoints only by admin var gUsername, gPassword string +var isDevelopment bool -func SetAdminCredentials() { +func SetAdminCredentials(devMode bool) { gUsername = viper.GetString("admin.username") gPassword = viper.GetString("admin.password") + isDevelopment = devMode } func AuthenticateAdmin(handler ReqRespHandlerf) ReqRespHandlerf { return func(w http.ResponseWriter, r *http.Request) { - uname, passwd, ok := r.BasicAuth() - if !ok { - w.WriteHeader(http.StatusForbidden) - w.Write([]byte("Admin only api")) // nolint - return - } + if !isDevelopment { + uname, passwd, ok := r.BasicAuth() + if !ok { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte("Admin only api")) // nolint + return + } - if uname != gUsername || passwd != gPassword { - w.WriteHeader(http.StatusForbidden) - w.Write([]byte("Invalid username or password")) // nolint - return + if uname != gUsername || passwd != gPassword { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte("Invalid username or password")) // nolint + return + } } handler(w, r) From 024cfa84588f6c883c609c61d61fc958000d3c70 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Tue, 24 Oct 2023 00:47:41 +0530 Subject: [PATCH 3/4] rmv listAlloc --- .../blobbercore/stats/blobberstats.go | 41 ------------------- 1 file changed, 41 deletions(-) diff --git a/code/go/0chain.net/blobbercore/stats/blobberstats.go b/code/go/0chain.net/blobbercore/stats/blobberstats.go index f4e0f1352..01877a44a 100644 --- a/code/go/0chain.net/blobbercore/stats/blobberstats.go +++ b/code/go/0chain.net/blobbercore/stats/blobberstats.go @@ -453,47 +453,6 @@ func (bs *BlobberStats) loadAllocationChallengeStats(ctx context.Context) { } } -func loadAllocationList(ctx context.Context) (interface{}, error) { - var ( - allocations = make([]AllocationId, 0) - db = datastore.GetStore().GetTransaction(ctx) - rows *sql.Rows - err error - ) - - rows, err = db.Table("reference_objects"). - Select("reference_objects.allocation_id"). - Group("reference_objects.allocation_id"). - Rows() - - if err != nil { - Logger.Error("Error in getting the allocation list", zap.Error(err)) - return nil, common.NewError("get_allocations_list_failed", - "Failed to get allocation list from DB") - } - defer rows.Close() - - for rows.Next() { - var allocationId AllocationId - if err = rows.Scan(&allocationId.Id); err != nil { - Logger.Error("Error in scanning record for blobber allocations", - zap.Error(err)) - return nil, common.NewError("get_allocations_list_failed", - "Failed to scan allocation from DB") - } - allocations = append(allocations, allocationId) - } - - if err = rows.Err(); err != nil && err != sql.ErrNoRows { - Logger.Error("Error in scanning record for blobber allocations", - zap.Error(err)) - return nil, common.NewError("get_allocations_list_failed", - "Failed to scan allocations from DB") - } - - return allocations, nil -} - type ReadMarkerEntity struct { ReadCounter int64 `gorm:"column:counter" json:"counter"` LatestRedeemedRC int64 `gorm:"column:latest_redeemed_rc"` From 1ccc26e455f899d0d5690c6e0f1293ce69ba9c0d Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Tue, 24 Oct 2023 19:24:53 +0530 Subject: [PATCH 4/4] build stats on start --- code/go/0chain.net/blobbercore/stats/blobberstats.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/code/go/0chain.net/blobbercore/stats/blobberstats.go b/code/go/0chain.net/blobbercore/stats/blobberstats.go index 01877a44a..61fdc410b 100644 --- a/code/go/0chain.net/blobbercore/stats/blobberstats.go +++ b/code/go/0chain.net/blobbercore/stats/blobberstats.go @@ -94,6 +94,12 @@ type AllocationId struct { func SetupStatsWorker(ctx context.Context) { fs = &BlobberStats{} go func() { + _ = datastore.GetStore().WithNewTransaction(func(ctx context.Context) error { + fs.loadBasicStats(ctx) + fs.loadDetailedStats(ctx) + fs.loadFailedChallengeList(ctx) + return common.NewError("rollback", "read_only") + }) for { select { case <-ctx.Done():