merge staging->main

.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+- package-ecosystem: "npm"
+  directory: "/frontend"
+  schedule:
+    interval: "weekly"
+    time: "05:00"
+    timezone: "Europe/Berlin"
+  target-branch: "staging"
+  assignees:
+    - "marcel-bitfly"
+    - "benji-bitfly"
+  commit-message:
+    prefix: "chore(update)"

.github/workflows/backend-linter.yml

@@ -37,7 +37,7 @@ jobs:
         with:
           version: v1.60.1
           working-directory: backend
-          args: --timeout=5m
+          args: --timeout=5m --config .golangci.yml
 
 
 

.github/workflows/backend-publish-docker.yml

@@ -10,6 +10,7 @@ on:
     branches:
       - main
       - staging
+      - staging-2
 
 # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
 env:

.github/workflows/frontend-publish-docker.yml

@@ -10,6 +10,7 @@ on:
     branches:
       - main
       - staging
+      - staging-2
 
 # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
 env:

backend/.golangci.yml

@@ -20,6 +20,7 @@ linters:
     - importas
     - unparam
   disable:
+    - testpackage
     - dupword
     - gocognit
     - dupl

backend/cmd/api/main.go

@@ -67,6 +67,7 @@ func Run() {
 	dummy := dataaccess.NewDummyService()
 
 	router := api.NewApiRouter(dataAccessor, dummy, cfg)
+	router.Use(version.HttpMiddleware)
 	router.Use(api.GetCorsMiddleware(cfg.CorsAllowedHosts))
 
 	if utils.Config.Metrics.Enabled {

backend/cmd/evm_node_indexer/main.go

@@ -19,22 +19,22 @@ import (
 	"sync/atomic"
 	"time"
 
+	gcp_bigtable "cloud.google.com/go/bigtable"
 	"github.com/ethereum/go-ethereum"
 	gethtypes "github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/gtuk/discordwebhook"
+	"github.com/lib/pq"
+	"golang.org/x/sync/errgroup"
+	"google.golang.org/api/option"
+
 	"github.com/gobitfly/beaconchain/pkg/commons/db"
 	"github.com/gobitfly/beaconchain/pkg/commons/hexutil"
 	"github.com/gobitfly/beaconchain/pkg/commons/log"
 	"github.com/gobitfly/beaconchain/pkg/commons/metrics"
 	"github.com/gobitfly/beaconchain/pkg/commons/types"
 	"github.com/gobitfly/beaconchain/pkg/commons/utils"
 	"github.com/gobitfly/beaconchain/pkg/commons/version"
-	"github.com/gtuk/discordwebhook"
-	"github.com/lib/pq"
-	"golang.org/x/sync/errgroup"
-	"google.golang.org/api/option"
-
-	gcp_bigtable "cloud.google.com/go/bigtable"
 )
 
 // defines
@@ -86,12 +86,18 @@ type intRange struct {
 
 // local globals
 var currentNodeBlockNumber atomic.Int64
-var elClient *ethclient.Client
+var elClient ethClient
 var reorgDepth *int64
 var httpClient *http.Client
 var errorIdentifier *regexp.Regexp
 var eth1RpcEndpoint string
 
+type ethClient interface {
+	SubscribeNewHead(ctx context.Context, ch chan<- *gethtypes.Header) (ethereum.Subscription, error)
+	ethereum.ChainIDReader
+	ethereum.BlockNumberReader
+}
+
 // init
 func init() {
 	httpClient = &http.Client{Timeout: time.Second * HTTP_TIMEOUT_IN_SECONDS}
@@ -275,7 +281,9 @@ func Run() {
 
 	// get latest block (as it's global, so we have a initial value)
 	log.Info("get latest block from node...")
-	updateBlockNumber(true, *noNewBlocks, time.Duration(*noNewBlocksThresholdSeconds)*time.Second, discordWebhookReportUrl, discordWebhookUser, discordWebhookAddTextFatal)
+	if err := updateBlockNumber(*noNewBlocks, time.Duration(*noNewBlocksThresholdSeconds)*time.Second, discordWebhookReportUrl, discordWebhookUser, discordWebhookAddTextFatal); err != nil {
+		log.Fatal(err, "updateBlockNumber", 0)
+	}
 	log.Infof("...get latest block (%s) from node done.", _formatInt64(currentNodeBlockNumber.Load()))
 
 	// //////////////////////////////////////////
@@ -892,21 +900,30 @@ func _splitAndVerifyJsonArray(jArray []byte, providedElementCount int64) ([][]by
 	return r, nil
 }
 
-// get newest block number from node, should be called always with TRUE
-func updateBlockNumber(firstCall bool, noNewBlocks bool, noNewBlocksThresholdDuration time.Duration, discordWebhookReportUrl *string, discordWebhookUser *string, discordWebhookAddTextFatal *string) {
-	if firstCall {
-		blockNumber, err := rpciGetLatestBlock()
-		if err != nil {
-			sendMessage(fmt.Sprintf("%s NODE EXPORT: Fatal, failed to get newest block from node, on first try %s", getChainNamePretty(), *discordWebhookAddTextFatal), discordWebhookReportUrl, discordWebhookUser)
-			log.Fatal(err, "fatal, failed to get newest block from node, on first try", 0)
-		}
-		currentNodeBlockNumber.Store(blockNumber)
-		if !noNewBlocks {
-			go updateBlockNumber(false, false, noNewBlocksThresholdDuration, discordWebhookReportUrl, discordWebhookUser, discordWebhookAddTextFatal)
-		}
-		return
+// get newest block number from node
+func updateBlockNumber(noNewBlocks bool, noNewBlocksThresholdDuration time.Duration, discordWebhookReportUrl *string, discordWebhookUser *string, discordWebhookAddTextFatal *string) error {
+	blockNumber, err := rpciGetLatestBlock()
+	if err != nil {
+		sendMessage(fmt.Sprintf("%s NODE EXPORT: Fatal, failed to get newest block from node, on first try %s", getChainNamePretty(), *discordWebhookAddTextFatal), discordWebhookReportUrl, discordWebhookUser)
+		log.Fatal(err, "fatal, failed to get newest block from node, on first try", 0)
 	}
+	currentNodeBlockNumber.Store(blockNumber)
+	// request the block number a second time to verify if node is not rewinding
+	blockNumber, err = rpciGetLatestBlock()
+	if err != nil {
+		return err
+	}
+	if blockNumber < currentNodeBlockNumber.Load() {
+		return fmt.Errorf("node is rewinding: newest block %d < previous block %d", blockNumber, currentNodeBlockNumber.Load())
+	}
+
+	if !noNewBlocks {
+		go monitorBlockNumber(noNewBlocksThresholdDuration, discordWebhookReportUrl, discordWebhookUser, discordWebhookAddTextFatal)
+	}
+	return nil
+}
 
+func monitorBlockNumber(noNewBlocksThresholdDuration time.Duration, discordWebhookReportUrl *string, discordWebhookUser *string, discordWebhookAddTextFatal *string) {
 	var errorText string
 	gotNewBlockAt := time.Now()
 	timePerBlock := time.Second * time.Duration(utils.Config.Chain.ClConfig.SecondsPerSlot)

backend/cmd/evm_node_indexer/main_test.go

@@ -0,0 +1,55 @@
+package evm_node_indexer
+
+import (
+	"context"
+	"math/big"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum"
+	gethtypes "github.com/ethereum/go-ethereum/core/types"
+
+	"github.com/gobitfly/beaconchain/pkg/commons/types"
+	"github.com/gobitfly/beaconchain/pkg/commons/utils"
+)
+
+func TestUpdateBlockNumberWithRewindingNode(t *testing.T) {
+	utils.Config = &types.Config{
+		Chain: types.Chain{
+			ClConfig: types.ClChainConfig{
+				SecondsPerSlot: 0,
+			},
+		},
+	}
+	elClient = &fakeEthClient{
+		chainID: big.NewInt(1),
+		blocks:  []uint64{126192682, 126186030},
+	}
+	err := updateBlockNumber(false, time.Second, nil, nil, nil)
+	if err == nil {
+		t.Fatal("expected error got nil")
+	}
+	if !strings.Contains(err.Error(), "node is rewinding") {
+		t.Fatalf("error message does not contain 'node is rewinding', got %s", err.Error())
+	}
+}
+
+type fakeEthClient struct {
+	chainID *big.Int
+	blocks  []uint64
+}
+
+func (f *fakeEthClient) SubscribeNewHead(ctx context.Context, ch chan<- *gethtypes.Header) (ethereum.Subscription, error) {
+	panic("implement me")
+}
+
+func (f *fakeEthClient) ChainID(ctx context.Context) (*big.Int, error) {
+	return f.chainID, nil
+}
+
+func (f *fakeEthClient) BlockNumber(ctx context.Context) (uint64, error) {
+	block := f.blocks[0]
+	f.blocks = f.blocks[1:]
+	return block, nil
+}

backend/pkg/api/api_test.go

@@ -298,8 +298,9 @@ func TestInternalSearchHandler(t *testing.T) {
 	}`)).Expect().Status(http.StatusOK).JSON().Decode(&resp)
 
 	assert.NotEqual(t, 0, len(resp.Data), "response data should not be empty")
-	assert.NotNil(t, resp.Data[0].NumValue, "validator index should not be nil")
-	assert.Equal(t, uint64(5), *resp.Data[0].NumValue, "validator index should be 5")
+	validatorByIndex, ok := resp.Data[0].Value.(api_types.SearchValidator)
+	assert.True(t, ok, "response data should be of type SearchValidator")
+	assert.Equal(t, uint64(5), validatorByIndex.Index, "validator index should be 5")
 
 	// search for validator by pubkey
 	resp = api_types.InternalPostSearchResponse{}
@@ -324,8 +325,9 @@ func TestInternalSearchHandler(t *testing.T) {
 	}`)).Expect().Status(http.StatusOK).JSON().Decode(&resp)
 
 	assert.NotEqual(t, 0, len(resp.Data), "response data should not be empty")
-	assert.NotNil(t, resp.Data[0].NumValue, "validator index should not be nil")
-	assert.Equal(t, uint64(5), *resp.Data[0].NumValue, "validator index should be 5")
+	validatorByPublicKey, ok := resp.Data[0].Value.(api_types.SearchValidator)
+	assert.True(t, ok, "response data should be of type SearchValidator")
+	assert.Equal(t, uint64(5), validatorByPublicKey.Index, "validator index should be 5")
 
 	// search for validator by withdawal address
 	resp = api_types.InternalPostSearchResponse{}
@@ -349,8 +351,9 @@ func TestInternalSearchHandler(t *testing.T) {
 	}`)).Expect().Status(http.StatusOK).JSON().Decode(&resp)
 
 	assert.NotEqual(t, 0, len(resp.Data), "response data should not be empty")
-	assert.NotNil(t, resp.Data[0].NumValue, "validator index should not be nil")
-	assert.Greater(t, *resp.Data[0].NumValue, uint64(0), "returned number of validators should be greater than 0")
+	validatorsByWithdrawalAddress, ok := resp.Data[0].Value.(api_types.SearchValidatorsByWithdrwalCredential)
+	assert.True(t, ok, "response data should be of type SearchValidator")
+	assert.Greater(t, validatorsByWithdrawalAddress.Count, uint64(0), "returned number of validators should be greater than 0")
 }
 
 func TestPublicAndSharedDashboards(t *testing.T) {

backend/pkg/api/data_access/data_access.go

@@ -190,6 +190,7 @@ func createDataAccessService(cfg *types.Config) *DataAccessService {
 				Port:         cfg.Frontend.WriterDatabase.Port,
 				MaxOpenConns: cfg.Frontend.WriterDatabase.MaxOpenConns,
 				MaxIdleConns: cfg.Frontend.WriterDatabase.MaxIdleConns,
+				SSL:          cfg.Frontend.WriterDatabase.SSL,
 			},
 			&types.DatabaseConfig{
 				Username:     cfg.Frontend.ReaderDatabase.Username,
@@ -199,6 +200,7 @@ func createDataAccessService(cfg *types.Config) *DataAccessService {
 				Port:         cfg.Frontend.ReaderDatabase.Port,
 				MaxOpenConns: cfg.Frontend.ReaderDatabase.MaxOpenConns,
 				MaxIdleConns: cfg.Frontend.ReaderDatabase.MaxIdleConns,
+				SSL:          cfg.Frontend.ReaderDatabase.SSL,
 			}, "pgx", "postgres",
 		)
 	}()

backend/pkg/api/data_access/dummy.go

@@ -274,7 +274,7 @@ func (d *DummyService) AddValidatorDashboardValidatorsByDepositAddress(ctx conte
 	return getDummyData[[]t.VDBPostValidatorsData](ctx)
 }
 
-func (d *DummyService) AddValidatorDashboardValidatorsByWithdrawalAddress(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, address string, limit uint64) ([]t.VDBPostValidatorsData, error) {
+func (d *DummyService) AddValidatorDashboardValidatorsByWithdrawalCredential(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, address string, limit uint64) ([]t.VDBPostValidatorsData, error) {
 	return getDummyData[[]t.VDBPostValidatorsData](ctx)
 }
 
@@ -514,16 +514,16 @@ func (d *DummyService) GetSearchValidatorsByDepositAddress(ctx context.Context,
 	return getDummyStruct[t.SearchValidatorsByDepositAddress](ctx)
 }
 
-func (d *DummyService) GetSearchValidatorsByDepositEnsName(ctx context.Context, chainId uint64, ensName string) (*t.SearchValidatorsByDepositEnsName, error) {
-	return getDummyStruct[t.SearchValidatorsByDepositEnsName](ctx)
+func (d *DummyService) GetSearchValidatorsByDepositEnsName(ctx context.Context, chainId uint64, ensName string) (*t.SearchValidatorsByDepositAddress, error) {
+	return getDummyStruct[t.SearchValidatorsByDepositAddress](ctx)
 }
 
 func (d *DummyService) GetSearchValidatorsByWithdrawalCredential(ctx context.Context, chainId uint64, credential []byte) (*t.SearchValidatorsByWithdrwalCredential, error) {
 	return getDummyStruct[t.SearchValidatorsByWithdrwalCredential](ctx)
 }
 
-func (d *DummyService) GetSearchValidatorsByWithdrawalEnsName(ctx context.Context, chainId uint64, ensName string) (*t.SearchValidatorsByWithrawalEnsName, error) {
-	return getDummyStruct[t.SearchValidatorsByWithrawalEnsName](ctx)
+func (d *DummyService) GetSearchValidatorsByWithdrawalEnsName(ctx context.Context, chainId uint64, ensName string) (*t.SearchValidatorsByWithdrwalCredential, error) {
+	return getDummyStruct[t.SearchValidatorsByWithdrwalCredential](ctx)
 }
 
 func (d *DummyService) GetSearchValidatorsByGraffiti(ctx context.Context, chainId uint64, graffiti string) (*t.SearchValidatorsByGraffiti, error) {
@@ -784,7 +784,7 @@ func (d *DummyService) PostUserMachineMetrics(ctx context.Context, userID uint64
 	return nil
 }
 
-func (d *DummyService) GetValidatorDashboardMobileValidators(ctx context.Context, dashboardId t.VDBId, period enums.TimePeriod, cursor string, colSort t.Sort[enums.VDBMobileValidatorsColumn], search string, limit uint64) ([]t.MobileValidatorDashboardValidatorsTableRow, *t.Paging, error) {
+func (d *DummyService) GetValidatorDashboardMobileValidators(ctx context.Context, dashboardId t.VDBId, groupId int64, period enums.TimePeriod, cursor string, colSort t.Sort[enums.VDBManageValidatorsColumn], search string, limit uint64) ([]t.MobileValidatorDashboardValidatorsTableRow, *t.Paging, error) {
 	return getDummyWithPaging[t.MobileValidatorDashboardValidatorsTableRow](ctx)
 }