From be99070e4cc50bf69f12c5b043ab8bc34033cde5 Mon Sep 17 00:00:00 2001 From: Manuel <5877862+manuelsc@users.noreply.github.com> Date: Tue, 19 Nov 2024 15:58:18 +0100 Subject: [PATCH] feat: add mobile validators endpoint with extra data fields - efficiency, rocket pool and current/next sync committee info Issue BEDS-966 --- backend/pkg/api/data_access/dummy.go | 2 +- backend/pkg/api/data_access/mobile.go | 185 +++++++++++++++++- .../api/enums/validator_dashboard_enums.go | 55 ------ backend/pkg/api/handlers/internal.go | 6 +- backend/pkg/api/types/mobile.go | 11 +- frontend/types/api/mobile.ts | 5 +- 6 files changed, 195 insertions(+), 69 deletions(-) diff --git a/backend/pkg/api/data_access/dummy.go b/backend/pkg/api/data_access/dummy.go index ebb280b1e..350642962 100644 --- a/backend/pkg/api/data_access/dummy.go +++ b/backend/pkg/api/data_access/dummy.go @@ -784,7 +784,7 @@ func (d *DummyService) PostUserMachineMetrics(ctx context.Context, userID uint64 return nil } -func (d *DummyService) GetValidatorDashboardMobileValidators(ctx context.Context, dashboardId t.VDBId, period enums.TimePeriod, cursor string, colSort t.Sort[enums.VDBMobileValidatorsColumn], search string, limit uint64) ([]t.MobileValidatorDashboardValidatorsTableRow, *t.Paging, error) { +func (d *DummyService) GetValidatorDashboardMobileValidators(ctx context.Context, dashboardId t.VDBId, groupId int64, period enums.TimePeriod, cursor string, colSort t.Sort[enums.VDBManageValidatorsColumn], search string, limit uint64) ([]t.MobileValidatorDashboardValidatorsTableRow, *t.Paging, error) { return getDummyWithPaging[t.MobileValidatorDashboardValidatorsTableRow](ctx) } diff --git a/backend/pkg/api/data_access/mobile.go b/backend/pkg/api/data_access/mobile.go index 373688439..89b2b7b03 100644 --- a/backend/pkg/api/data_access/mobile.go +++ b/backend/pkg/api/data_access/mobile.go @@ -7,11 +7,14 @@ import ( "time" "github.com/doug-martin/goqu/v9" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/gobitfly/beaconchain/pkg/api/enums" t "github.com/gobitfly/beaconchain/pkg/api/types" + "github.com/gobitfly/beaconchain/pkg/commons/cache" "github.com/gobitfly/beaconchain/pkg/commons/utils" constypes "github.com/gobitfly/beaconchain/pkg/consapi/types" "github.com/gobitfly/beaconchain/pkg/userservice" + "github.com/lib/pq" "github.com/pkg/errors" "github.com/shopspring/decimal" "golang.org/x/sync/errgroup" @@ -27,7 +30,7 @@ type AppRepository interface { AddMobilePurchase(ctx context.Context, tx *sql.Tx, userID uint64, paymentDetails t.MobileSubscription, verifyResponse *userservice.VerifyResponse, extSubscriptionId string) error GetLatestBundleForNativeVersion(ctx context.Context, nativeVersion uint64) (*t.MobileAppBundleStats, error) IncrementBundleDeliveryCount(ctx context.Context, bundleVerison uint64) error - GetValidatorDashboardMobileValidators(ctx context.Context, dashboardId t.VDBId, period enums.TimePeriod, cursor string, colSort t.Sort[enums.VDBMobileValidatorsColumn], search string, limit uint64) ([]t.MobileValidatorDashboardValidatorsTableRow, *t.Paging, error) + GetValidatorDashboardMobileValidators(ctx context.Context, dashboardId t.VDBId, groupId int64, period enums.TimePeriod, cursor string, colSort t.Sort[enums.VDBManageValidatorsColumn], search string, limit uint64) ([]t.MobileValidatorDashboardValidatorsTableRow, *t.Paging, error) } // GetUserIdByRefreshToken basically used to confirm the claimed user id with the refresh token. Returns the userId if successful @@ -363,6 +366,182 @@ func (d *DataAccessService) getInternalRpNetworkStats(ctx context.Context) (*t.R return &networkStats, err } -func (d *DataAccessService) GetValidatorDashboardMobileValidators(ctx context.Context, dashboardId t.VDBId, period enums.TimePeriod, cursor string, colSort t.Sort[enums.VDBMobileValidatorsColumn], search string, limit uint64) ([]t.MobileValidatorDashboardValidatorsTableRow, *t.Paging, error) { - return d.dummy.GetValidatorDashboardMobileValidators(ctx, dashboardId, period, cursor, colSort, search, limit) +func (d *DataAccessService) GetValidatorDashboardMobileValidators(ctx context.Context, dashboardId t.VDBId, groupId int64, period enums.TimePeriod, cursor string, colSort t.Sort[enums.VDBManageValidatorsColumn], search string, limit uint64) ([]t.MobileValidatorDashboardValidatorsTableRow, *t.Paging, error) { + result, p, err := d.GetValidatorDashboardValidators(ctx, dashboardId, groupId, cursor, colSort, search, limit) + if err != nil { + return nil, p, err + } + + // Get extra information for this result subset + validatorMapping, err := d.services.GetCurrentValidatorMapping() + if err != nil { + return nil, nil, errors.Wrap(err, "validator mapping error") + } + + pubKeys := make([][]byte, 0, len(result)) + indices := make([]uint64, 0, len(result)) + for _, row := range result { + metadata := validatorMapping.ValidatorMetadata[row.Index] + pubKeys = append(pubKeys, metadata.PublicKey) + indices = append(indices, row.Index) + } + + wg := errgroup.Group{} + + type RocketPoolData struct { + PubKey []byte `db:"pubkey"` + Commission float64 `db:"node_fee"` + PenaltyCount uint64 `db:"penalty_count"` + DepositAmount decimal.Decimal `db:"node_deposit_balance"` + Status string `db:"status"` + IsInSmoothingPool bool `db:"smoothing_pool_opted_in"` + } + + var rocketPoolMap map[uint64]RocketPoolData + wg.Go(func() error { + rocketPoolResults := []RocketPoolData{} + + validatorsQuery := ` + SELECT + pubkey, + node_fee, + penalty_count, + node_deposit_balance, + status, + rn.smoothing_pool_opted_in + FROM rocketpool_minipools + LEFT JOIN rocketpool_nodes rn ON rocketpool_minipools.node_address = rn.address + WHERE pubkey = ANY($1) + ` + err := d.alloyReader.SelectContext(ctx, &rocketPoolResults, validatorsQuery, pq.ByteaArray(pubKeys)) + if err != nil { + return errors.Wrap(err, "error retrieving rocketpool data") + } + + rocketPoolMap = make(map[uint64]RocketPoolData, len(rocketPoolResults)) + for _, row := range rocketPoolResults { + validatorIndex := validatorMapping.ValidatorIndices[string(t.PubKey(hexutil.Encode(row.PubKey)))] + rocketPoolMap[validatorIndex] = row + } + return nil + }) + + var efficienciesMap map[uint64]float64 + wg.Go(func() error { + var err error + clickhouseTable, _, err := d.getTablesForPeriod(period) + if err != nil { + return err + } + + efficienciesMap, err = d.getIndividualEfficiencies(ctx, indices, clickhouseTable) + if err != nil { + return errors.Wrap(err, "error retrieving efficiencies") + } + return nil + }) + + currentSyncCommitteeValidators := make(map[uint64]bool) + upcomingSyncCommitteeValidators := make(map[uint64]bool) + wg.Go(func() error { + latestEpoch := cache.LatestEpoch.Get() + var err error + currentSyncCommitteeValidators, upcomingSyncCommitteeValidators, err = d.getCurrentAndUpcomingSyncCommittees(ctx, latestEpoch) + if err != nil { + return errors.Wrap(err, "error retrieving sync committees") + } + return nil + }) + + err = wg.Wait() + if err != nil { + return nil, nil, err + } + + var mobileResult []t.MobileValidatorDashboardValidatorsTableRow + for _, row := range result { + mobileRow := t.MobileValidatorDashboardValidatorsTableRow{ + Index: row.Index, + PublicKey: row.PublicKey, + GroupId: row.GroupId, + Balance: row.Balance, + Status: row.Status, + QueuePosition: row.QueuePosition, + WithdrawalCredential: row.WithdrawalCredential, + IsInSyncCommittee: currentSyncCommitteeValidators[row.Index], + IsInNextSyncCommittee: upcomingSyncCommitteeValidators[row.Index], + Efficiency: efficienciesMap[row.Index], + } + + if rp, ok := rocketPoolMap[row.Index]; ok { + mobileRow.RocketPool = &t.MobileValidatorDashboardValidatorsRocketPool{ + DepositAmount: rp.DepositAmount, + Commission: rp.Commission, + Status: rp.Status, + PenaltyCount: rp.PenaltyCount, + IsInSmoothingPool: rp.IsInSmoothingPool, + } + } + + mobileResult = append(mobileResult, mobileRow) + } + + return mobileResult, p, nil +} + +func (d *DataAccessService) getIndividualEfficiencies(ctx context.Context, indices []uint64, table string) (map[uint64]float64, error) { + ds := goqu.Dialect("postgres"). + From(goqu.L(fmt.Sprintf(`%s AS r FINAL`, table))). + Select( + goqu.L("r.validator_index"), + goqu.L("COALESCE(r.attestations_reward::decimal, 0) AS attestations_reward"), + goqu.L("COALESCE(r.attestations_ideal_reward::decimal, 0) AS attestations_ideal_reward"), + goqu.L("COALESCE(r.blocks_proposed, 0) AS blocks_proposed"), + goqu.L("COALESCE(r.blocks_scheduled, 0) AS blocks_scheduled"), + goqu.L("COALESCE(r.sync_executed, 0) AS sync_executed"), + goqu.L("COALESCE(r.sync_scheduled, 0) AS sync_scheduled"), + ).Where(goqu.L("r.validator_index IN ?", indices)) + + var queryResult []struct { + Index uint64 `db:"validator_index"` + AttestationReward decimal.Decimal `db:"attestations_reward"` + AttestationIdealReward decimal.Decimal `db:"attestations_ideal_reward"` + BlocksProposed uint64 `db:"blocks_proposed"` + BlocksScheduled uint64 `db:"blocks_scheduled"` + SyncExecuted uint64 `db:"sync_executed"` + SyncScheduled uint64 `db:"sync_scheduled"` + } + + query, args, err := ds.Prepared(true).ToSQL() + if err != nil { + return nil, fmt.Errorf("error preparing query: %w", err) + } + + err = d.clickhouseReader.SelectContext(ctx, &queryResult, query, args...) + if err != nil { + return nil, err + } + + result := make(map[uint64]float64, len(queryResult)) + + // Calculate efficiency + for _, row := range queryResult { + var attestationEfficiency, proposerEfficiency, syncEfficiency sql.NullFloat64 + if !row.AttestationIdealReward.IsZero() { + attestationEfficiency.Float64 = row.AttestationReward.Div(row.AttestationIdealReward).InexactFloat64() + attestationEfficiency.Valid = true + } + if row.BlocksScheduled > 0 { + proposerEfficiency.Float64 = float64(row.BlocksProposed) / float64(row.BlocksScheduled) + proposerEfficiency.Valid = true + } + if row.SyncScheduled > 0 { + syncEfficiency.Float64 = float64(row.SyncExecuted) / float64(row.SyncScheduled) + syncEfficiency.Valid = true + } + + result[row.Index] = utils.CalculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency) + } + + return result, nil } diff --git a/backend/pkg/api/enums/validator_dashboard_enums.go b/backend/pkg/api/enums/validator_dashboard_enums.go index c241ef98e..6a4ab4945 100644 --- a/backend/pkg/api/enums/validator_dashboard_enums.go +++ b/backend/pkg/api/enums/validator_dashboard_enums.go @@ -298,61 +298,6 @@ var VDBManageValidatorsColumns = struct { VDBManageValidatorsWithdrawalCredential, } -// ---------------- -// Validator Dashboard Manage Validators Table - -type VDBMobileValidatorsColumn int - -var _ EnumFactory[VDBMobileValidatorsColumn] = VDBMobileValidatorsColumn(0) - -const ( - VDBMobileValidatorsIndex VDBMobileValidatorsColumn = iota - VDBMobileValidatorsPublicKey - VDBMobileValidatorsBalance - VDBMobileValidatorsStatus - VDBMobileValidatorsWithdrawalCredential - VDBMobileValidatorsEfficiency -) - -func (c VDBMobileValidatorsColumn) Int() int { - return int(c) -} - -func (VDBMobileValidatorsColumn) NewFromString(s string) VDBMobileValidatorsColumn { - switch s { - case "index": - return VDBMobileValidatorsIndex - case "public_key": - return VDBMobileValidatorsPublicKey - case "balance": - return VDBMobileValidatorsBalance - case "status": - return VDBMobileValidatorsStatus - case "withdrawal_credential": - return VDBMobileValidatorsWithdrawalCredential - case "efficiency": - return VDBMobileValidatorsEfficiency - default: - return VDBMobileValidatorsColumn(-1) - } -} - -var VDBMobileValidatorsColumns = struct { - Index VDBManageValidatorsColumn - PublicKey VDBManageValidatorsColumn - Balance VDBManageValidatorsColumn - Status VDBManageValidatorsColumn - WithdrawalCredential VDBManageValidatorsColumn - Efficiency VDBMobileValidatorsColumn -}{ - VDBManageValidatorsIndex, - VDBManageValidatorsPublicKey, - VDBManageValidatorsBalance, - VDBManageValidatorsStatus, - VDBManageValidatorsWithdrawalCredential, - VDBMobileValidatorsEfficiency, -} - // ---------------- // Validator Dashboard Archived Reasons diff --git a/backend/pkg/api/handlers/internal.go b/backend/pkg/api/handlers/internal.go index 4fbabc2b2..d04b3f4f6 100644 --- a/backend/pkg/api/handlers/internal.go +++ b/backend/pkg/api/handlers/internal.go @@ -381,14 +381,14 @@ func (h *HandlerService) InternalGetValidatorDashboardMobileValidators(w http.Re } q := r.URL.Query() pagingParams := v.checkPagingParams(q) - + groupId := v.checkGroupId(q.Get("group_id"), allowEmpty) period := checkEnum[enums.TimePeriod](&v, q.Get("period"), "period") - sort := checkSort[enums.VDBMobileValidatorsColumn](&v, q.Get("sort")) + sort := checkSort[enums.VDBManageValidatorsColumn](&v, q.Get("sort")) if v.hasErrors() { handleErr(w, r, v) return } - data, paging, err := h.daService.GetValidatorDashboardMobileValidators(r.Context(), *dashboardId, period, pagingParams.cursor, *sort, pagingParams.search, pagingParams.limit) + data, paging, err := h.daService.GetValidatorDashboardMobileValidators(r.Context(), *dashboardId, groupId, period, pagingParams.cursor, *sort, pagingParams.search, pagingParams.limit) if err != nil { handleErr(w, r, err) return diff --git a/backend/pkg/api/types/mobile.go b/backend/pkg/api/types/mobile.go index 41a5e21d5..059239f62 100644 --- a/backend/pkg/api/types/mobile.go +++ b/backend/pkg/api/types/mobile.go @@ -23,11 +23,11 @@ type MobileWidgetData struct { type InternalGetValidatorDashboardMobileWidgetResponse ApiDataResponse[MobileWidgetData] type MobileValidatorDashboardValidatorsRocketPool struct { - DepositAmount decimal.Decimal `json:"deposit_Amount"` + DepositAmount decimal.Decimal `json:"deposit_amount"` Commission float64 `json:"commission"` // percentage, 0-1 Status string `json:"status" tstype:"'Staking' | 'Dissolved' | 'Prelaunch' | 'Initialized' | 'Withdrawable'" faker:"oneof: Staking, Dissolved, Prelaunch, Initialized, Withdrawable"` PenaltyCount uint64 `json:"penalty_count"` - IsInSmoothingPool bool `json:"is_in_smokaothing_pool"` + IsInSmoothingPool bool `json:"is_in_smoothing_pool"` } type MobileValidatorDashboardValidatorsTableRow struct { Index uint64 `json:"index"` @@ -38,9 +38,10 @@ type MobileValidatorDashboardValidatorsTableRow struct { QueuePosition *uint64 `json:"queue_position,omitempty"` WithdrawalCredential Hash `json:"withdrawal_credential"` // additional mobile fields - IsInSyncCommittee bool `json:"is_in_sync_committee"` - Efficiency float64 `json:"efficiency"` - RocketPool *MobileValidatorDashboardValidatorsRocketPool `json:"rocket_pool,omitempty"` + IsInSyncCommittee bool `json:"is_in_sync_committee"` + IsInNextSyncCommittee bool `json:"is_in_next_sync_committee"` + Efficiency float64 `json:"efficiency"` + RocketPool *MobileValidatorDashboardValidatorsRocketPool `json:"rocket_pool,omitempty"` } type InternalGetValidatorDashboardMobileValidatorsResponse ApiPagingResponse[MobileValidatorDashboardValidatorsTableRow] diff --git a/frontend/types/api/mobile.ts b/frontend/types/api/mobile.ts index 2166ba1dd..3b7bfc484 100644 --- a/frontend/types/api/mobile.ts +++ b/frontend/types/api/mobile.ts @@ -22,11 +22,11 @@ export interface MobileWidgetData { } export type InternalGetValidatorDashboardMobileWidgetResponse = ApiDataResponse; export interface MobileValidatorDashboardValidatorsRocketPool { - deposit_Amount: string /* decimal.Decimal */; + deposit_amount: string /* decimal.Decimal */; commission: number /* float64 */; // percentage, 0-1 status: 'Staking' | 'Dissolved' | 'Prelaunch' | 'Initialized' | 'Withdrawable'; penalty_count: number /* uint64 */; - is_in_smokaothing_pool: boolean; + is_in_smoothing_pool: boolean; } export interface MobileValidatorDashboardValidatorsTableRow { index: number /* uint64 */; @@ -40,6 +40,7 @@ export interface MobileValidatorDashboardValidatorsTableRow { * additional mobile fields */ is_in_sync_committee: boolean; + is_in_next_sync_committee: boolean; efficiency: number /* float64 */; rocket_pool?: MobileValidatorDashboardValidatorsRocketPool; }