Feat/sovereign

.github/workflows/deploy-docker.yml

@@ -3,6 +3,7 @@ name: Publish Docker image
 on:
   release:
     types: [published]
+  pull_request:
 
 jobs:
   push_to_registry:
@@ -31,24 +32,25 @@ jobs:
           fi
 
       - name: Log in to Docker Hub
+        if: ${{ github.event_name == 'release' && github.event.action == 'published' }}
         uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
+        if: ${{ github.event_name == 'release' && github.event.action == 'published' }}
         id: meta
         uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
         with:
-          images: multiversx/elastic-indexer
+          images: multiversx/sovereign-elastic-indexer
 
       - name: Build and push Docker image
         id: push
         uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
         with:
           context: .
-          file: ./Dockerfile
-          push: true
+          file: ./Dockerfile-sovereign
+          push: ${{ github.event_name == 'release' && github.event.action == 'published' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-

Dockerfile-sovereign

@@ -0,0 +1,26 @@
+FROM golang:1.20.7 as builder
+
+RUN apt-get update && apt-get install -y
+
+WORKDIR /multiversx
+COPY . .
+
+WORKDIR /multiversx/cmd/elasticindexer
+
+RUN go build -o elasticindexer
+
+# ===== SECOND STAGE ======
+FROM ubuntu:22.04
+RUN apt-get update && apt-get install -y
+
+RUN useradd -m -u 1000 appuser
+USER appuser
+
+COPY --from=builder --chown=appuser /multiversx/cmd/elasticindexer /multiversx
+
+EXPOSE 22111
+
+WORKDIR /multiversx
+
+ENTRYPOINT ["./elasticindexer", "--sovereign"]
+CMD ["--log-level", "*:DEBUG"]

Makefile

@@ -36,9 +36,16 @@ integration-tests-open-search:
 INDEXER_IMAGE_NAME="elasticindexer"
 INDEXER_IMAGE_TAG="latest"
 DOCKER_FILE=Dockerfile
+SOVEREIGN_DOCKER_FILE=Dockerfile-sovereign
 
 docker-build:
 	docker build \
 		 -t ${INDEXER_IMAGE_NAME}:${INDEXER_IMAGE_TAG} \
 		 -f ${DOCKER_FILE} \
 		 .
+
+docker-sovereign-build:
+	docker build \
+		 -t ${INDEXER_IMAGE_NAME}:${INDEXER_IMAGE_TAG} \
+		 -f ${SOVEREIGN_DOCKER_FILE} \
+		 .

client/disabled/elasticClient.go

@@ -0,0 +1,78 @@
+package disabled
+
+import (
+	"bytes"
+	"context"
+)
+
+type elasticClient struct{}
+
+// NewDisabledElasticClient -
+func NewDisabledElasticClient() *elasticClient {
+	return &elasticClient{}
+}
+
+// DoBulkRequest -
+func (ec *elasticClient) DoBulkRequest(_ context.Context, _ *bytes.Buffer, _ string) error {
+	return nil
+}
+
+// DoQueryRemove -
+func (ec *elasticClient) DoQueryRemove(_ context.Context, _ string, _ *bytes.Buffer) error {
+	return nil
+}
+
+// DoMultiGet -
+func (ec *elasticClient) DoMultiGet(_ context.Context, _ []string, _ string, _ bool, _ interface{}) error {
+	return nil
+}
+
+// DoScrollRequest -
+func (ec *elasticClient) DoScrollRequest(_ context.Context, _ string, _ []byte, _ bool, _ func(responseBytes []byte) error) error {
+	return nil
+}
+
+// DoCountRequest -
+func (ec *elasticClient) DoCountRequest(_ context.Context, _ string, _ []byte) (uint64, error) {
+	return 0, nil
+}
+
+// UpdateByQuery -
+func (ec *elasticClient) UpdateByQuery(_ context.Context, _ string, _ *bytes.Buffer) error {
+	return nil
+}
+
+// PutMappings -
+func (ec *elasticClient) PutMappings(_ string, _ *bytes.Buffer) error {
+	return nil
+}
+
+// CheckAndCreateIndex -
+func (ec *elasticClient) CheckAndCreateIndex(_ string) error {
+	return nil
+}
+
+// CheckAndCreateAlias -
+func (ec *elasticClient) CheckAndCreateAlias(_ string, _ string) error {
+	return nil
+}
+
+// CheckAndCreateTemplate -
+func (ec *elasticClient) CheckAndCreateTemplate(_ string, _ *bytes.Buffer) error {
+	return nil
+}
+
+// CheckAndCreatePolicy -
+func (ec *elasticClient) CheckAndCreatePolicy(_ string, _ *bytes.Buffer) error {
+	return nil
+}
+
+// IsEnabled -
+func (ec *elasticClient) IsEnabled() bool {
+	return false
+}
+
+// IsInterfaceNil - returns true if there is no value under the interface
+func (ec *elasticClient) IsInterfaceNil() bool {
+	return ec == nil
+}

client/disabled/elasticClient_test.go

@@ -0,0 +1,31 @@
+package disabled
+
+import (
+	"bytes"
+	"context"
+	"testing"
+
+	"github.com/multiversx/mx-chain-core-go/core/check"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDisabledElasticClient_MethodsShouldNotPanic(t *testing.T) {
+	t.Parallel()
+
+	ec := NewDisabledElasticClient()
+	require.False(t, check.IfNil(ec))
+
+	require.NotPanics(t, func() {
+		_ = ec.DoBulkRequest(context.Background(), new(bytes.Buffer), "")
+		_ = ec.DoQueryRemove(context.Background(), "", new(bytes.Buffer))
+		_ = ec.DoMultiGet(context.Background(), make([]string, 0), "", true, nil)
+		_ = ec.DoScrollRequest(context.Background(), "", []byte(""), true, nil)
+		_, _ = ec.DoCountRequest(context.Background(), "", []byte(""))
+		_ = ec.UpdateByQuery(context.Background(), "", new(bytes.Buffer))
+		_ = ec.PutMappings("", new(bytes.Buffer))
+		_ = ec.CheckAndCreateIndex("")
+		_ = ec.CheckAndCreateAlias("", "")
+		_ = ec.CheckAndCreateTemplate("", new(bytes.Buffer))
+		_ = ec.CheckAndCreatePolicy("", new(bytes.Buffer))
+	})
+}

client/elasticClientCommon.go

@@ -6,11 +6,14 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math"
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/elastic/go-elasticsearch/v7/esapi"
+
 	"github.com/multiversx/mx-chain-es-indexer-go/data"
 	"github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer"
 )
@@ -268,3 +271,11 @@ func parseResponse(res *esapi.Response, dest interface{}, errorHandler responseE
 
 	return nil
 }
+
+// RetryBackOff returns elastic retry backoff duration
+func RetryBackOff(attempt int) time.Duration {
+	d := time.Duration(math.Exp2(float64(attempt))) * time.Second
+	log.Debug("elastic: retry backoff", "attempt", attempt, "sleep duration", d)
+
+	return d
+}

client/mainChainElasticClient.go

@@ -0,0 +1,35 @@
+package client
+
+import (
+	"github.com/multiversx/mx-chain-core-go/core/check"
+
+	"github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer"
+	"github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc"
+)
+
+type mainChainElasticClient struct {
+	elasticproc.DatabaseClientHandler
+	indexingEnabled bool
+}
+
+// NewMainChainElasticClient creates a new sovereign elastic client
+func NewMainChainElasticClient(esClient elasticproc.DatabaseClientHandler, indexingEnabled bool) (*mainChainElasticClient, error) {
+	if check.IfNil(esClient) {
+		return nil, dataindexer.ErrNilDatabaseClient
+	}
+
+	return &mainChainElasticClient{
+		esClient,
+		indexingEnabled,
+	}, nil
+}
+
+// IsEnabled returns true if main chain elastic client is enabled
+func (mcec *mainChainElasticClient) IsEnabled() bool {
+	return mcec.indexingEnabled
+}
+
+// IsInterfaceNil returns true if there is no value under the interface
+func (mcec *mainChainElasticClient) IsInterfaceNil() bool {
+	return mcec == nil
+}

client/mainChainElasticClient_test.go

@@ -0,0 +1,42 @@
+package client
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/elastic/go-elasticsearch/v7"
+	"github.com/stretchr/testify/require"
+
+	"github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer"
+)
+
+func TestNewMainChainElasticClient(t *testing.T) {
+	t.Run("nil elastic client, should error", func(t *testing.T) {
+		mainChainESClient, err := NewMainChainElasticClient(nil, true)
+		require.Error(t, err, dataindexer.ErrNilDatabaseClient)
+		require.True(t, mainChainESClient.IsInterfaceNil())
+	})
+	t.Run("valid elastic client, should work", func(t *testing.T) {
+		esClient, err := NewElasticClient(elasticsearch.Config{
+			Addresses: []string{"http://localhost:9200"},
+		})
+		require.Nil(t, err)
+		require.NotNil(t, esClient)
+
+		mainChainESClient, err := NewMainChainElasticClient(esClient, true)
+		require.NoError(t, err)
+		require.Equal(t, "*client.mainChainElasticClient", fmt.Sprintf("%T", mainChainESClient))
+	})
+}
+
+func TestMainChainElasticClient_IsEnabled(t *testing.T) {
+	esClient, err := NewElasticClient(elasticsearch.Config{
+		Addresses: []string{"http://localhost:9200"},
+	})
+	require.Nil(t, err)
+	require.NotNil(t, esClient)
+
+	mainChainESClient, err := NewMainChainElasticClient(esClient, true)
+	require.NoError(t, err)
+	require.Equal(t, true, mainChainESClient.IsEnabled())
+}

cmd/elasticindexer/config/config.toml

@@ -4,6 +4,7 @@
         "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags",
         "logs", "delegators", "operations", "esdts", "values", "events"
     ]
+    esdt-prefix = ""
     [config.address-converter]
         length = 32
         type = "bech32"

cmd/elasticindexer/config/prefs.toml

@@ -23,3 +23,11 @@
         username = ""
         password = ""
         bulk-request-max-size-in-bytes = 4194304 # 4MB
+
+    # Configuration for main chain elastic cluster
+    # Used by the sovereign chain indexer to index incoming new tokens properties
+    [config.main-chain-elastic-cluster]
+        enabled = true
+        url = "http://localhost:9201"
+        username = ""
+        password = ""

cmd/elasticindexer/flags.go

@@ -47,4 +47,9 @@ var (
 		Name:  "disable-ansi-color",
 		Usage: "Boolean option for disabling ANSI colors in the logging system.",
 	}
+	// sovereign defines a flag that specifies if the es instance should run for a sovereign chain
+	sovereign = cli.BoolFlag{
+		Name:  "sovereign",
+		Usage: "If set to true, will use sovereign run type components",
+	}
 )

cmd/elasticindexer/main.go

@@ -11,13 +11,14 @@ import (
 	"github.com/multiversx/mx-chain-core-go/core/check"
 	"github.com/multiversx/mx-chain-core-go/core/closing"
 	"github.com/multiversx/mx-chain-core-go/data/outport"
+	logger "github.com/multiversx/mx-chain-logger-go"
+	"github.com/multiversx/mx-chain-logger-go/file"
+	"github.com/urfave/cli"
+
 	"github.com/multiversx/mx-chain-es-indexer-go/config"
 	"github.com/multiversx/mx-chain-es-indexer-go/factory"
 	"github.com/multiversx/mx-chain-es-indexer-go/metrics"
 	"github.com/multiversx/mx-chain-es-indexer-go/process/wsindexer"
-	logger "github.com/multiversx/mx-chain-logger-go"
-	"github.com/multiversx/mx-chain-logger-go/file"
-	"github.com/urfave/cli"
 )
 
 var (
@@ -63,6 +64,7 @@ func main() {
 		logLevel,
 		logSaveFile,
 		disableAnsiColor,
+		sovereign,
 	}
 	app.Authors = []cli.Author{
 		{
@@ -86,6 +88,7 @@ func startIndexer(ctx *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("%w while loading the config file", err)
 	}
+	cfg.Sovereign = ctx.GlobalBool(sovereign.Name)
 
 	clusterCfg, err := loadClusterConfig(ctx.GlobalString(configurationPreferencesFile.Name))
 	if err != nil {

config/config.go

@@ -4,6 +4,7 @@ package config
 type Config struct {
 	Config struct {
 		AvailableIndices []string `toml:"available-indices"`
+		ESDTPrefix       string   `toml:"esdt-prefix"`
 		AddressConverter struct {
 			Length int    `toml:"length"`
 			Type   string `toml:"type"`
@@ -29,6 +30,7 @@ type Config struct {
 			LogsPath             string `toml:"logs-path"`
 		} `toml:"logs"`
 	} `toml:"config"`
+	Sovereign bool
 }
 
 // ClusterConfig will hold the config for the Elasticsearch cluster
@@ -51,6 +53,12 @@ type ClusterConfig struct {
 			Password                  string `toml:"password"`
 			BulkRequestMaxSizeInBytes int    `toml:"bulk-request-max-size-in-bytes"`
 		} `toml:"elastic-cluster"`
+		MainChainCluster struct {
+			Enabled  bool   `toml:"enabled"`
+			URL      string `toml:"url"`
+			UserName string `toml:"username"`
+			Password string `toml:"password"`
+		} `toml:"main-chain-elastic-cluster"`
 	} `toml:"config"`
 }