From be03bd270a26480996b87893d462f59786bc5e9c Mon Sep 17 00:00:00 2001 From: tanlang Date: Thu, 28 Sep 2023 16:13:21 +0800 Subject: [PATCH 1/4] feat: add more metrics --- dagstore/wrapper.go | 25 +++++++++++ go.mod | 2 +- go.sum | 4 +- metrics/exporter.go | 45 ------------------- metrics/metrics.go | 19 +++++++- metrics/modules.go | 3 +- .../provider_datatransfer_sub.go | 3 ++ 7 files changed, 50 insertions(+), 51 deletions(-) delete mode 100644 metrics/exporter.go diff --git a/dagstore/wrapper.go b/dagstore/wrapper.go index cec53541..84da3e42 100644 --- a/dagstore/wrapper.go +++ b/dagstore/wrapper.go @@ -19,6 +19,7 @@ import ( "github.com/filecoin-project/go-statemachine/fsm" "github.com/ipfs-force-community/droplet/v2/config" + "github.com/ipfs-force-community/droplet/v2/metrics" "github.com/ipfs-force-community/droplet/v2/models/badger" "github.com/ipfs-force-community/droplet/v2/models/repo" carindex "github.com/ipld/go-car/v2/index" @@ -127,6 +128,30 @@ func NewDAGStore(ctx context.Context, return nil, nil, fmt.Errorf("failed to create DAG store: %w", err) } + // thread for metrics + go func() { + tick := time.NewTicker(1 * time.Minute) + defer tick.Stop() + for { + select { + case <-tick.C: + infos := dagst.AllShardsInfo() + stateCount := make(map[dagstore.ShardState]int) + for _, info := range infos { + if _, ok := stateCount[info.ShardState]; !ok { + stateCount[info.ShardState] = 0 + } + stateCount[info.ShardState]++ + } + for state, count := range stateCount { + metrics.ShardNum.Set(ctx, state.String(), int64(count)) + } + case <-ctx.Done(): + return + } + } + }() + w := &Wrapper{ cfg: cfg, dagst: dagst, diff --git a/go.mod b/go.mod index 0d52de8e..cb3adf5f 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e github.com/hashicorp/go-multierror v1.1.1 github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef - github.com/ipfs-force-community/metrics v1.0.1-0.20231011024528-8c881d456601 + github.com/ipfs-force-community/metrics v1.0.1-0.20231122034306-983501e1959d github.com/ipfs-force-community/sophon-auth v1.14.0 github.com/ipfs-force-community/sophon-gateway v1.14.0 github.com/ipfs-force-community/sophon-messager v1.14.0 diff --git a/go.sum b/go.sum index 28fd2569..3ee6e7dc 100644 --- a/go.sum +++ b/go.sum @@ -698,8 +698,8 @@ github.com/ipfs-force-community/go-fil-markets v1.2.6-0.20230822060005-aee2cbae5 github.com/ipfs-force-community/go-fil-markets v1.2.6-0.20230822060005-aee2cbae5b01/go.mod h1:eryxo/oVgIxaR5g5CNr9PlvZOi+u/bak0IsPL/PT1hk= github.com/ipfs-force-community/go-jsonrpc v0.1.9 h1:5QavBltfvV6fz/+EbYsCkVxJ1MSJncZm6YuPs1SLdZU= github.com/ipfs-force-community/go-jsonrpc v0.1.9/go.mod h1:jBSvPTl8V1N7gSTuCR4bis8wnQnIjHbRPpROol6iQKM= -github.com/ipfs-force-community/metrics v1.0.1-0.20231011024528-8c881d456601 h1:zxKQ30KAD6KfvSFAx9tuqQXLDsEHyF+eVaUBXXYC2bU= -github.com/ipfs-force-community/metrics v1.0.1-0.20231011024528-8c881d456601/go.mod h1:wM6EmkEcnJgWOFcVytgvK0u15awEmt8He0f2kAdsFDA= +github.com/ipfs-force-community/metrics v1.0.1-0.20231122034306-983501e1959d h1:EJQONl9uXat91SpVWE575+Xqyx14eRfTNjCHfdSYwKQ= +github.com/ipfs-force-community/metrics v1.0.1-0.20231122034306-983501e1959d/go.mod h1:wM6EmkEcnJgWOFcVytgvK0u15awEmt8He0f2kAdsFDA= github.com/ipfs-force-community/sophon-auth v1.14.0 h1:ctBJ6UHkcytEzfVPgiiHo0cW4FGQrE7r1H3Um0FcHbo= github.com/ipfs-force-community/sophon-auth v1.14.0/go.mod h1:d6J6u3zyIwcEajRho5BhVBcoIChEf0K76wP4yJEfEhc= github.com/ipfs-force-community/sophon-gateway v1.14.0 h1:nl4RnYiwVwviYbi0qhdisUHqHaOdoux1BbT6ps/F1iY= diff --git a/metrics/exporter.go b/metrics/exporter.go deleted file mode 100644 index ef8c4555..00000000 --- a/metrics/exporter.go +++ /dev/null @@ -1,45 +0,0 @@ -package metrics - -import ( - "context" - "fmt" - - "github.com/ipfs-force-community/metrics" - logging "github.com/ipfs/go-log/v2" - "go.opencensus.io/stats/view" -) - -var log = logging.Logger("metrics") - -func SetupMetrics(ctx context.Context, metricsConfig *metrics.MetricsConfig) error { - log.Infof("metrics config: enabled: %v, exporter type: %s, prometheus: %+v, graphite: %+v", - metricsConfig.Enabled, metricsConfig.Exporter.Type, metricsConfig.Exporter.Prometheus, - metricsConfig.Exporter.Graphite) - - if !metricsConfig.Enabled { - return nil - } - - if err := view.Register(views...); err != nil { - return fmt.Errorf("cannot register the view: %w", err) - } - - switch metricsConfig.Exporter.Type { - case metrics.ETPrometheus: - go func() { - if err := metrics.RegisterPrometheusExporter(ctx, metricsConfig.Exporter.Prometheus); err != nil { - log.Errorf("failed to register prometheus exporter err: %v", err) - return - } - log.Infof("prometheus exporter server graceful shutdown successful") - }() - - case metrics.ETGraphite: - if err := metrics.RegisterGraphiteExporter(ctx, metricsConfig.Exporter.Graphite); err != nil { - log.Errorf("failed to register graphite exporter: %v", err) - } - default: - log.Warnf("invalid exporter type: %s", metricsConfig.Exporter.Type) - } - return nil -} diff --git a/metrics/metrics.go b/metrics/metrics.go index 1085835b..e8e725fe 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -1,7 +1,8 @@ package metrics import ( - "github.com/filecoin-project/go-jsonrpc/metrics" + rpcMetrics "github.com/filecoin-project/go-jsonrpc/metrics" + "github.com/ipfs-force-community/metrics" "go.opencensus.io/stats" "go.opencensus.io/stats/view" "go.opencensus.io/tag" @@ -12,6 +13,12 @@ var ( StorageNameTag, _ = tag.NewKey("storage") ) +var ( + ApiState = metrics.NewInt64("api/state", "api service state. 0: down, 1: up", "") + RetrievalTransferEvent = metrics.NewCounterWithCategory("retrieval/transfer_event", "retrieval transfer event") + ShardNum = metrics.NewInt64WithCategory("shard/num", "shard num in different state", "") +) + var ( GraphsyncReceivingPeersCount = stats.Int64("graphsync/receiving_peers", "number of peers we are receiving graphsync data from", stats.UnitDimensionless) GraphsyncReceivingActiveCount = stats.Int64("graphsync/receiving_active", "number of active receiving graphsync transfers", stats.UnitDimensionless) @@ -127,4 +134,12 @@ var views = append([]*view.View{ StorageRetrievalHitCountView, StorageSaveHitCountView, -}, metrics.DefaultViews...) +}, rpcMetrics.DefaultViews...) + +func init() { + for _, v := range views { + if err := view.Register(v); err != nil { + panic(err) + } + } +} diff --git a/metrics/modules.go b/metrics/modules.go index bf66ee48..30273b31 100644 --- a/metrics/modules.go +++ b/metrics/modules.go @@ -18,7 +18,8 @@ var MetricsOpts = func(scope string, metricsConfig *metrics.MetricsConfig) build return metrics2.CtxScope(context.Background(), scope) }), builder.Override(startMetricsKey, func(mctx metrics.MetricsCtx, lc fx.Lifecycle) error { - return SetupMetrics(metrics.LifecycleCtx(mctx, lc), metricsConfig) + ctx := metrics.LifecycleCtx(mctx, lc) + return metrics.SetupMetrics(ctx, metricsConfig) }), ) } diff --git a/retrievalprovider/provider_datatransfer_sub.go b/retrievalprovider/provider_datatransfer_sub.go index a33cd623..316e8771 100644 --- a/retrievalprovider/provider_datatransfer_sub.go +++ b/retrievalprovider/provider_datatransfer_sub.go @@ -6,6 +6,7 @@ import ( datatransfer "github.com/filecoin-project/go-data-transfer/v2" rm "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/ipfs-force-community/droplet/v2/metrics" ) // ProviderDataTransferSubscriber is the function called when an event occurs in a data @@ -39,6 +40,8 @@ func ProviderDataTransferSubscriber(deals IDatatransferHandler) datatransfer.Sub } } + metrics.RetrievalTransferEvent.Tick(ctx, event.Code.String()) + switch event.Code { case datatransfer.Accept: mlog = mlog.With("retrievalEvent", rm.ProviderEvents[rm.ProviderEventDealAccepted]) From f6a1e0001659b78c901c1f1d927402bd67f84ef1 Mon Sep 17 00:00:00 2001 From: tanlang Date: Thu, 28 Sep 2023 16:17:23 +0800 Subject: [PATCH 2/4] feat: disable lint rules unused-parameter --- .golangci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.golangci.yml b/.golangci.yml index 48bf4fbb..f36c2d13 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -43,6 +43,10 @@ issues: linters-settings: goconst: min-occurrences: 6 + revive: + rules: + - name: unused-parameter + disabled: true run: skip-dirs-use-default: false From 9c9df8b0fd27c211c8c4c7cb7a2aeaed1b1d1ac8 Mon Sep 17 00:00:00 2001 From: tanlang Date: Thu, 7 Dec 2023 09:26:40 +0800 Subject: [PATCH 3/4] feat: add api state metrics --- rpc/rpc.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rpc/rpc.go b/rpc/rpc.go index c7b9a988..153135a6 100644 --- a/rpc/rpc.go +++ b/rpc/rpc.go @@ -21,6 +21,7 @@ import ( "github.com/ipfs-force-community/sophon-auth/jwtclient" "github.com/ipfs-force-community/droplet/v2/config" + "github.com/ipfs-force-community/droplet/v2/metrics" "github.com/ipfs-force-community/droplet/v2/retrievalprovider/httpretrieval" ) @@ -89,6 +90,7 @@ func ServeRPC( log.Errorf("shutting down RPC server failed: %s", err) } log.Warn("RPC Graceful shutdown successful") + metrics.ApiState.Set(ctx, 0) }() addr, err := multiaddr.NewMultiaddr(apiCfg.ListenAddress) @@ -100,6 +102,7 @@ func ServeRPC( if err != nil { return err } + metrics.ApiState.Set(ctx, 1) log.Infof("start rpc listen %s", addr) if err := srv.Serve(manet.NetListener(nl)); err != nil && err != http.ErrServerClosed { From 312c929dcc0f16d2200910482db0290972cb253c Mon Sep 17 00:00:00 2001 From: tanlang Date: Thu, 7 Dec 2023 16:29:30 +0800 Subject: [PATCH 4/4] chore: update metrics mod --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index cb3adf5f..1ffb7883 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e github.com/hashicorp/go-multierror v1.1.1 github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef - github.com/ipfs-force-community/metrics v1.0.1-0.20231122034306-983501e1959d + github.com/ipfs-force-community/metrics v1.0.1-0.20231207081445-30178e706d09 github.com/ipfs-force-community/sophon-auth v1.14.0 github.com/ipfs-force-community/sophon-gateway v1.14.0 github.com/ipfs-force-community/sophon-messager v1.14.0 diff --git a/go.sum b/go.sum index 3ee6e7dc..4d86259f 100644 --- a/go.sum +++ b/go.sum @@ -698,8 +698,8 @@ github.com/ipfs-force-community/go-fil-markets v1.2.6-0.20230822060005-aee2cbae5 github.com/ipfs-force-community/go-fil-markets v1.2.6-0.20230822060005-aee2cbae5b01/go.mod h1:eryxo/oVgIxaR5g5CNr9PlvZOi+u/bak0IsPL/PT1hk= github.com/ipfs-force-community/go-jsonrpc v0.1.9 h1:5QavBltfvV6fz/+EbYsCkVxJ1MSJncZm6YuPs1SLdZU= github.com/ipfs-force-community/go-jsonrpc v0.1.9/go.mod h1:jBSvPTl8V1N7gSTuCR4bis8wnQnIjHbRPpROol6iQKM= -github.com/ipfs-force-community/metrics v1.0.1-0.20231122034306-983501e1959d h1:EJQONl9uXat91SpVWE575+Xqyx14eRfTNjCHfdSYwKQ= -github.com/ipfs-force-community/metrics v1.0.1-0.20231122034306-983501e1959d/go.mod h1:wM6EmkEcnJgWOFcVytgvK0u15awEmt8He0f2kAdsFDA= +github.com/ipfs-force-community/metrics v1.0.1-0.20231207081445-30178e706d09 h1:qEI6ItxKtgOupMMuGJwqK5zEzztKKPUP1QKq9g+X5bM= +github.com/ipfs-force-community/metrics v1.0.1-0.20231207081445-30178e706d09/go.mod h1:wM6EmkEcnJgWOFcVytgvK0u15awEmt8He0f2kAdsFDA= github.com/ipfs-force-community/sophon-auth v1.14.0 h1:ctBJ6UHkcytEzfVPgiiHo0cW4FGQrE7r1H3Um0FcHbo= github.com/ipfs-force-community/sophon-auth v1.14.0/go.mod h1:d6J6u3zyIwcEajRho5BhVBcoIChEf0K76wP4yJEfEhc= github.com/ipfs-force-community/sophon-gateway v1.14.0 h1:nl4RnYiwVwviYbi0qhdisUHqHaOdoux1BbT6ps/F1iY=