From a900c92ef070e14a879eb51e3f6c6d0aaf4d35d9 Mon Sep 17 00:00:00 2001 From: Eslam-Nawara Date: Wed, 4 Sep 2024 16:52:25 +0300 Subject: [PATCH 1/4] add all available endpoints to zos environment --- pkg/environment/environment.go | 66 +++++++++++++++++++++--------- pkg/perf/healthcheck/network.go | 6 +-- pkg/perf/iperf/iperf_task.go | 14 ++++++- pkg/perf/publicip/publicip_task.go | 29 +++++++++---- pkg/registrar/register.go | 7 +++- pkg/registrar/registrar.go | 7 +++- 6 files changed, 95 insertions(+), 34 deletions(-) diff --git a/pkg/environment/environment.go b/pkg/environment/environment.go index 6888565aa..50fca3f6f 100644 --- a/pkg/environment/environment.go +++ b/pkg/environment/environment.go @@ -48,8 +48,8 @@ type Environment struct { // SINCE RELAYS FOR A NODE IS STORED ON THE CHAIN IN A LIMITED SPACE // PLEASE MAKE SURE THAT ANY ENV HAS NO MORE THAN FOUR RELAYS CONFIGURED RelayURL []string - ActivationURL string - GraphQL string + ActivationURL []string + GraphQL []string // private vlan to join // if set, zos will use this as its priv vlan @@ -108,30 +108,44 @@ var ( RunningMode: RunningDev, SubstrateURL: []string{ "wss://tfchain.dev.grid.tf/", + "wss://tfchain.02.dev.grid.tf", }, RelayURL: []string{ "wss://relay.dev.grid.tf", "wss://relay.02.dev.grid.tf", }, - ActivationURL: "https://activation.dev.grid.tf/activation/activate", - FlistURL: "redis://hub.grid.tf:9900", - BinRepo: "tf-zos-v3-bins.dev", - GraphQL: "https://graphql.dev.grid.tf/graphql", + ActivationURL: []string{ + "https://activation.dev.grid.tf/activation/activate", + "https://activation.02.dev.grid.tf/activation/activate", + }, + FlistURL: "redis://hub.grid.tf:9900", + BinRepo: "tf-zos-v3-bins.dev", + GraphQL: []string{ + "https://graphql.dev.grid.tf/graphql", + "https://graphql.02.dev.grid.tf/graphql", + }, } envTest = Environment{ RunningMode: RunningTest, SubstrateURL: []string{ "wss://tfchain.test.grid.tf/", + "wss://tfchain.02.test.grid.tf", }, RelayURL: []string{ "wss://relay.test.grid.tf", "wss://relay.02.test.grid.tf", }, - ActivationURL: "https://activation.test.grid.tf/activation/activate", - FlistURL: "redis://hub.grid.tf:9900", - BinRepo: "tf-zos-v3-bins.test", - GraphQL: "https://graphql.test.grid.tf/graphql", + ActivationURL: []string{ + "https://activation.test.grid.tf/activation/activate", + "https://activation.02.test.grid.tf/activation/activate", + }, + FlistURL: "redis://hub.grid.tf:9900", + BinRepo: "tf-zos-v3-bins.test", + GraphQL: []string{ + "https://graphql.test.grid.tf/graphql", + "https://graphql.02.test.grid.tf/graphql", + }, } envQA = Environment{ @@ -142,17 +156,25 @@ var ( }, RelayURL: []string{ "wss://relay.qa.grid.tf", + "wss://relay.02.qa.grid.tf", + }, + ActivationURL: []string{ + "https://activation.qa.grid.tf/activation/activate", + "https://activation.02.qa.grid.tf/activation/activate", + }, + FlistURL: "redis://hub.grid.tf:9900", + BinRepo: "tf-zos-v3-bins.qanet", + GraphQL: []string{ + "https://graphql.qa.grid.tf/graphql", + "https://graphql.02.qa.grid.tf/graphql", }, - ActivationURL: "https://activation.qa.grid.tf/activation/activate", - FlistURL: "redis://hub.grid.tf:9900", - BinRepo: "tf-zos-v3-bins.qanet", - GraphQL: "https://graphql.qa.grid.tf/graphql", } envProd = Environment{ RunningMode: RunningMain, SubstrateURL: []string{ "wss://tfchain.grid.tf/", + "wss://tfchain.02.grid.tf", "wss://02.tfchain.grid.tf/", "wss://03.tfchain.grid.tf/", "wss://04.tfchain.grid.tf/", @@ -161,10 +183,16 @@ var ( "wss://relay.grid.tf", "wss://relay.02.grid.tf", }, - ActivationURL: "https://activation.grid.tf/activation/activate", - FlistURL: "redis://hub.grid.tf:9900", - BinRepo: "tf-zos-v3-bins", - GraphQL: "https://graphql.grid.tf/graphql", + ActivationURL: []string{ + "https://activation.grid.tf/activation/activate", + "https://activation.02.grid.tf/activation/activate", + }, + FlistURL: "redis://hub.grid.tf:9900", + BinRepo: "tf-zos-v3-bins", + GraphQL: []string{ + "https://graphql.grid.tf/graphql", + "https://graphql.02.grid.tf/graphql", + }, } ) @@ -241,7 +269,7 @@ func getEnvironmentFromParams(params kernel.Params) (Environment, error) { if activation, ok := params.Get("activation"); ok { if len(activation) > 0 { - env.ActivationURL = activation[len(activation)-1] + env.ActivationURL = activation } } diff --git a/pkg/perf/healthcheck/network.go b/pkg/perf/healthcheck/network.go index 1ab10c89c..9a99eb49a 100644 --- a/pkg/perf/healthcheck/network.go +++ b/pkg/perf/healthcheck/network.go @@ -17,10 +17,10 @@ const defaultRequestTimeout = 5 * time.Second func networkCheck(ctx context.Context) []error { env := environment.MustGet() - servicesUrl := []string{ - env.ActivationURL, env.GraphQL, env.FlistURL, - } + servicesUrl := []string{env.FlistURL} + servicesUrl = append(append(servicesUrl, env.SubstrateURL...), env.RelayURL...) + servicesUrl = append(append(servicesUrl, env.ActivationURL...), env.GraphQL...) var errors []error diff --git a/pkg/perf/iperf/iperf_task.go b/pkg/perf/iperf/iperf_task.go index 002b525d6..d48528e2b 100644 --- a/pkg/perf/iperf/iperf_task.go +++ b/pkg/perf/iperf/iperf_task.go @@ -76,10 +76,20 @@ func (t *IperfTest) Jitter() uint32 { // Run runs the tcp test and returns the result func (t *IperfTest) Run(ctx context.Context) (interface{}, error) { env := environment.MustGet() - g := graphql.NewGraphQl(env.GraphQL) + var ( + g graphql.GraphQl + err error + freeFarmNodes []graphql.Node + ) // get public up nodes - freeFarmNodes, err := g.GetUpNodes(ctx, 0, 1, 0, true, true) + for _, url := range env.GraphQL { + g = graphql.NewGraphQl(url) + freeFarmNodes, err = g.GetUpNodes(ctx, 0, 1, 0, true, true) + if err == nil { + break + } + } if err != nil { return nil, errors.Wrap(err, "failed to list freefarm nodes from graphql") } diff --git a/pkg/perf/publicip/publicip_task.go b/pkg/perf/publicip/publicip_task.go index 5b68c1740..8a433f9b3 100644 --- a/pkg/perf/publicip/publicip_task.go +++ b/pkg/perf/publicip/publicip_task.go @@ -34,11 +34,15 @@ const ( FetchRealIPFailed = "failed to get real public IP to the node" ) -var errPublicIPLookup = errors.New("failed to reach public ip service") -var errSkippedValidating = errors.New("skipped, there is a node with less ID available") +var ( + errPublicIPLookup = errors.New("failed to reach public ip service") + errSkippedValidating = errors.New("skipped, there is a node with less ID available") +) -const testMacvlan = "pub" -const testNamespace = "pubtestns" +const ( + testMacvlan = "pub" + testNamespace = "pubtestns" +) type publicIPValidationTask struct{} @@ -96,7 +100,6 @@ func (p *publicIPValidationTask) Run(ctx context.Context) (interface{}, error) { report, err = p.validateIPs(farm.PublicIPs) return err }) - if err != nil { return nil, fmt.Errorf("failed to run public IP validation: %w", err) } @@ -179,12 +182,23 @@ func (p *publicIPValidationTask) validateIPs(publicIPs []substrate.PublicIP) (ma func isLeastValidNode(ctx context.Context, farmID uint32, substrateGateway *stubs.SubstrateGatewayStub) (bool, error) { env := environment.MustGet() - gql := graphql.NewGraphQl(env.GraphQL) + var ( + nodes []graphql.Node + err error + ) + + for _, url := range env.GraphQL { + gql := graphql.NewGraphQl(url) - nodes, err := gql.GetUpNodes(ctx, 0, farmID, 0, false, false) + nodes, err = gql.GetUpNodes(ctx, 0, farmID, 0, false, false) + if err == nil { + break + } + } if err != nil { return false, fmt.Errorf("failed to get farm %d nodes: %w", farmID, err) } + cl := perf.GetZbusClient(ctx) registrar := stubs.NewRegistrarStub(cl) var nodeID uint32 @@ -196,7 +210,6 @@ func isLeastValidNode(ctx context.Context, farmID uint32, substrateGateway *stub } return nil }, backoff.NewConstantBackOff(10*time.Second)) - if err != nil { return false, fmt.Errorf("failed to get node id: %w", err) } diff --git a/pkg/registrar/register.go b/pkg/registrar/register.go index 891d2ee37..d4c14b631 100644 --- a/pkg/registrar/register.go +++ b/pkg/registrar/register.go @@ -149,7 +149,12 @@ func registerNode( sk := ed25519.PrivateKey(mgr.PrivateKey(ctx)) - if _, err := substrateGateway.EnsureAccount(ctx, env.ActivationURL, tcUrl, tcHash); err != nil { + for _, url := range env.ActivationURL { + if _, err = substrateGateway.EnsureAccount(ctx, url, tcUrl, tcHash); err == nil { + break + } + } + if err != nil { return 0, 0, errors.Wrap(err, "failed to ensure account") } diff --git a/pkg/registrar/registrar.go b/pkg/registrar/registrar.go index 25e6756db..774f8668d 100644 --- a/pkg/registrar/registrar.go +++ b/pkg/registrar/registrar.go @@ -155,8 +155,13 @@ func (r *Registrar) register(ctx context.Context, cl zbus.Client, env environmen func (r *Registrar) reActivate(ctx context.Context, cl zbus.Client, env environment.Environment) error { substrateGateway := stubs.NewSubstrateGatewayStub(cl) + var err error - _, err := substrateGateway.EnsureAccount(ctx, env.ActivationURL, tcUrl, tcHash) + for _, url := range env.ActivationURL { + if _, err = substrateGateway.EnsureAccount(ctx, url, tcUrl, tcHash); err == nil { + break + } + } return err } From 88fae6c25315ee02cc330e5276fae7295132c3d3 Mon Sep 17 00:00:00 2001 From: Eslam-Nawara Date: Thu, 5 Sep 2024 15:27:01 +0300 Subject: [PATCH 2/4] refactor multiple activation service url --- pkg/api_gateway.go | 2 +- pkg/perf/iperf/iperf_task.go | 1 + pkg/perf/publicip/publicip_task.go | 2 +- pkg/registrar/register.go | 7 +------ pkg/registrar/registrar.go | 7 +------ pkg/stubs/api_gateway_stub.go | 2 +- pkg/substrate_gateway/substrate_gateway.go | 11 ++++++++--- 7 files changed, 14 insertions(+), 18 deletions(-) diff --git a/pkg/api_gateway.go b/pkg/api_gateway.go index ba6f6b1c1..58ce6df10 100644 --- a/pkg/api_gateway.go +++ b/pkg/api_gateway.go @@ -10,7 +10,7 @@ import ( type SubstrateGateway interface { CreateNode(node substrate.Node) (uint32, error) CreateTwin(relay string, pk []byte) (uint32, error) - EnsureAccount(activationURL string, termsAndConditionsLink string, termsAndConditionsHash string) (info substrate.AccountInfo, err error) + EnsureAccount(activationURL []string, termsAndConditionsLink string, termsAndConditionsHash string) (info substrate.AccountInfo, err error) GetContract(id uint64) (substrate.Contract, SubstrateError) GetContractIDByNameRegistration(name string) (uint64, SubstrateError) GetFarm(id uint32) (substrate.Farm, error) diff --git a/pkg/perf/iperf/iperf_task.go b/pkg/perf/iperf/iperf_task.go index 6cf665068..df4a7f80e 100644 --- a/pkg/perf/iperf/iperf_task.go +++ b/pkg/perf/iperf/iperf_task.go @@ -81,6 +81,7 @@ func (t *IperfTest) Run(ctx context.Context) (interface{}, error) { return nil, err } + // get public up nodes freeFarmNodes, err := g.GetUpNodes(ctx, 0, 1, 0, true, true) if err != nil { return nil, errors.Wrap(err, "failed to list freefarm nodes from graphql") diff --git a/pkg/perf/publicip/publicip_task.go b/pkg/perf/publicip/publicip_task.go index b81a36526..9b8499e94 100644 --- a/pkg/perf/publicip/publicip_task.go +++ b/pkg/perf/publicip/publicip_task.go @@ -186,11 +186,11 @@ func isLeastValidNode(ctx context.Context, farmID uint32, substrateGateway *stub if err != nil { return false, err } + nodes, err := gql.GetUpNodes(ctx, 0, farmID, 0, false, false) if err != nil { return false, fmt.Errorf("failed to get farm %d nodes: %w", farmID, err) } - cl := perf.GetZbusClient(ctx) registrar := stubs.NewRegistrarStub(cl) var nodeID uint32 diff --git a/pkg/registrar/register.go b/pkg/registrar/register.go index d4c14b631..891d2ee37 100644 --- a/pkg/registrar/register.go +++ b/pkg/registrar/register.go @@ -149,12 +149,7 @@ func registerNode( sk := ed25519.PrivateKey(mgr.PrivateKey(ctx)) - for _, url := range env.ActivationURL { - if _, err = substrateGateway.EnsureAccount(ctx, url, tcUrl, tcHash); err == nil { - break - } - } - if err != nil { + if _, err := substrateGateway.EnsureAccount(ctx, env.ActivationURL, tcUrl, tcHash); err != nil { return 0, 0, errors.Wrap(err, "failed to ensure account") } diff --git a/pkg/registrar/registrar.go b/pkg/registrar/registrar.go index 774f8668d..25e6756db 100644 --- a/pkg/registrar/registrar.go +++ b/pkg/registrar/registrar.go @@ -155,13 +155,8 @@ func (r *Registrar) register(ctx context.Context, cl zbus.Client, env environmen func (r *Registrar) reActivate(ctx context.Context, cl zbus.Client, env environment.Environment) error { substrateGateway := stubs.NewSubstrateGatewayStub(cl) - var err error - for _, url := range env.ActivationURL { - if _, err = substrateGateway.EnsureAccount(ctx, url, tcUrl, tcHash); err == nil { - break - } - } + _, err := substrateGateway.EnsureAccount(ctx, env.ActivationURL, tcUrl, tcHash) return err } diff --git a/pkg/stubs/api_gateway_stub.go b/pkg/stubs/api_gateway_stub.go index 1fbdf0984..dbd340cfa 100644 --- a/pkg/stubs/api_gateway_stub.go +++ b/pkg/stubs/api_gateway_stub.go @@ -63,7 +63,7 @@ func (s *SubstrateGatewayStub) CreateTwin(ctx context.Context, arg0 string, arg1 return } -func (s *SubstrateGatewayStub) EnsureAccount(ctx context.Context, arg0 string, arg1 string, arg2 string) (ret0 tfchainclientgo.AccountInfo, ret1 error) { +func (s *SubstrateGatewayStub) EnsureAccount(ctx context.Context, arg0 []string, arg1 string, arg2 string) (ret0 tfchainclientgo.AccountInfo, ret1 error) { args := []interface{}{arg0, arg1, arg2} result, err := s.client.RequestContext(ctx, s.module, s.object, "EnsureAccount", args...) if err != nil { diff --git a/pkg/substrate_gateway/substrate_gateway.go b/pkg/substrate_gateway/substrate_gateway.go index 7bd936970..62be310d6 100644 --- a/pkg/substrate_gateway/substrate_gateway.go +++ b/pkg/substrate_gateway/substrate_gateway.go @@ -48,16 +48,21 @@ func (g *substrateGateway) CreateTwin(relay string, pk []byte) (uint32, error) { return g.sub.CreateTwin(g.identity, relay, pk) } -func (g *substrateGateway) EnsureAccount(activationURL string, termsAndConditionsLink string, termsAndConditionsHash string) (info substrate.AccountInfo, err error) { +func (g *substrateGateway) EnsureAccount(activationURL []string, termsAndConditionsLink string, termsAndConditionsHash string) (info substrate.AccountInfo, err error) { log.Debug(). Str("method", "EnsureAccount"). - Str("activation url", activationURL). + Strs("activation url", activationURL). Str("terms and conditions link", termsAndConditionsLink). Str("terms and conditions hash", termsAndConditionsHash). Msg("method called") g.mu.Lock() defer g.mu.Unlock() - return g.sub.EnsureAccount(g.identity, activationURL, termsAndConditionsLink, termsAndConditionsHash) + for _, url := range activationURL { + if info, err = g.sub.EnsureAccount(g.identity, url, termsAndConditionsLink, termsAndConditionsHash); err == nil { + return + } + } + return } func (g *substrateGateway) GetContract(id uint64) (result substrate.Contract, serr pkg.SubstrateError) { From 29842b8f7367568025f3d93991b651fe49f53ac3 Mon Sep 17 00:00:00 2001 From: Eslam-Nawara Date: Sun, 8 Sep 2024 13:08:27 +0300 Subject: [PATCH 3/4] retry ensureAccount only on ActivationServiceError --- pkg/substrate_gateway/substrate_gateway.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/substrate_gateway/substrate_gateway.go b/pkg/substrate_gateway/substrate_gateway.go index 62be310d6..c9bca2e31 100644 --- a/pkg/substrate_gateway/substrate_gateway.go +++ b/pkg/substrate_gateway/substrate_gateway.go @@ -58,9 +58,13 @@ func (g *substrateGateway) EnsureAccount(activationURL []string, termsAndConditi g.mu.Lock() defer g.mu.Unlock() for _, url := range activationURL { - if info, err = g.sub.EnsureAccount(g.identity, url, termsAndConditionsLink, termsAndConditionsHash); err == nil { - return + info, err = g.sub.EnsureAccount(g.identity, url, termsAndConditionsLink, termsAndConditionsHash) + // check other activationURL only if EnsureAccount failed with ActivationServiceError + if errors.Is(err, substrate.ActivationServiceError{}) { + log.Debug().Str("activation url", url).Err(err).Msg("failed to EnsureAccount with ActivationServiceError") + continue } + return } return } From 124739428f2dc12c60d978ee722e06e7ebb612b3 Mon Sep 17 00:00:00 2001 From: Eslam-Nawara Date: Mon, 9 Sep 2024 16:21:03 +0300 Subject: [PATCH 4/4] check for error of type ActivationServiceError --- pkg/substrate_gateway/substrate_gateway.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/substrate_gateway/substrate_gateway.go b/pkg/substrate_gateway/substrate_gateway.go index c9bca2e31..8a7d7fe7e 100644 --- a/pkg/substrate_gateway/substrate_gateway.go +++ b/pkg/substrate_gateway/substrate_gateway.go @@ -57,14 +57,14 @@ func (g *substrateGateway) EnsureAccount(activationURL []string, termsAndConditi Msg("method called") g.mu.Lock() defer g.mu.Unlock() + for _, url := range activationURL { info, err = g.sub.EnsureAccount(g.identity, url, termsAndConditionsLink, termsAndConditionsHash) // check other activationURL only if EnsureAccount failed with ActivationServiceError - if errors.Is(err, substrate.ActivationServiceError{}) { - log.Debug().Str("activation url", url).Err(err).Msg("failed to EnsureAccount with ActivationServiceError") - continue + if err == nil || !errors.As(err, &substrate.ActivationServiceError{}) { + return } - return + log.Debug().Str("activation url", url).Err(err).Msg("failed to EnsureAccount with ActivationServiceError") } return }