From e2329756167652d54f090d1c89f75c30d725d77f Mon Sep 17 00:00:00 2001 From: MohammadReza Palide Date: Wed, 29 Nov 2023 14:08:48 +0000 Subject: [PATCH 1/7] add --sk to services for available dmsghttp during e2e and integration env test --- docker/docker-compose.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 728098d4..2dd0ffb8 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -167,7 +167,7 @@ services: depends_on: - tpd-redis - postgres-db - entrypoint: "/release/transport-discovery --pg-host postgres-db --pg-port 5432 --redis redis://tpd-redis:6379" + entrypoint: "/release/transport-discovery --pg-host postgres-db --pg-port 5432 --redis redis://tpd-redis:6379 --sk 85789cd7efcdf35340c8f4cb3e15bccdff57efda49af5bcdddaae57374d25ab8" stdin_open: true # docker run -i tty: true # docker run -t @@ -190,7 +190,7 @@ services: - "9092:9092" depends_on: - transport-discovery - entrypoint: "/release/route-finder --pg-host postgres-db --pg-port 5432" + entrypoint: "/release/route-finder --pg-host postgres-db --pg-port 5432 --sk 640905f9fd60d6bf1ee88519f61035765d65319d9f3bac6c5f2fc14ea5cf15ca" stdin_open: true # docker run -i tty: true # docker run -t @@ -209,7 +209,7 @@ services: depends_on: - dmsgd-redis - transport-discovery - entrypoint: "/usr/local/bin/dmsg-discovery --redis redis://dmsgd-redis:6379 -t" + entrypoint: "/usr/local/bin/dmsg-discovery --redis redis://dmsgd-redis:6379 --sk b3f6706cb72215d3873ef92cc0c6037a47fe651112b1685017d6347eed0fb714 -t" stdin_open: true # docker run -i tty: true # docker run -t @@ -279,7 +279,7 @@ services: - PG_USER=postgres - PG_PASSWORD=postgres - PG_DATABASE=postgres - entrypoint: "/release/service-discovery --pg-host postgres-db --pg-port 5432 --redis redis://sd-redis:6379" + entrypoint: "/release/service-discovery --pg-host postgres-db --pg-port 5432 --redis redis://sd-redis:6379 --sk ccc3ae9993ed03bed80b1f9feb747d7c6ca3342a65c3923191b7527c2c00ef0f" stdin_open: true # docker run -i tty: true # docker run -t @@ -303,7 +303,7 @@ services: - PG_DATABASE=postgres ports: - "9096:9096" - entrypoint: "/release/uptime-tracker --pg-host postgres-db --pg-port 5432 --redis redis://ut-redis:6379" + entrypoint: "/release/uptime-tracker --pg-host postgres-db --pg-port 5432 --redis redis://ut-redis:6379 --sk e20681b4fd1eb1904ebfbf8fba79a138bbac481e1c3f77bcfab3a877525fda32" stdin_open: true # docker run -i tty: true # docker run -t @@ -322,7 +322,7 @@ services: depends_on: - ar-redis restart: on-failure - entrypoint: "/release/address-resolver --redis redis://ar-redis:6379" + entrypoint: "/release/address-resolver --redis redis://ar-redis:6379 --sk 1fa0a7b80438b8d31655b5c69efd57bcf931ac828438ff2925ab36e4abcc426a" stdin_open: true # docker run -i tty: true # docker run -t From d11f79f922de50c9f54f7363b7079d0414b7d199 Mon Sep 17 00:00:00 2001 From: MohammadReza Palide Date: Thu, 30 Nov 2023 18:24:12 +0000 Subject: [PATCH 2/7] update dmsg@develp --- go.mod | 2 +- go.sum | 2 ++ vendor/github.com/skycoin/dmsg/pkg/dmsg/client.go | 10 ++++++++++ vendor/modules.txt | 2 +- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 785fcd02..15db2fed 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/json-iterator/go v1.1.12 github.com/rs/cors v1.8.2 github.com/sirupsen/logrus v1.9.3 - github.com/skycoin/dmsg v1.3.12-0.20231120175607-0b6d5135bbc5 + github.com/skycoin/dmsg v1.3.12-0.20231130180155-65d707cd1b01 github.com/skycoin/skywire v1.3.14-0.20231120181804-7c5994927a04 github.com/skycoin/skywire-utilities v0.0.0-20231120175000-12be4345eb26 github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 diff --git a/go.sum b/go.sum index ebb8da48..ae223e5e 100644 --- a/go.sum +++ b/go.sum @@ -390,6 +390,8 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skycoin/dmsg v1.3.12-0.20231120175607-0b6d5135bbc5 h1:yGTInFb9SqchM5w0RrFA4vgq9tzhKQStURpetUor+4s= github.com/skycoin/dmsg v1.3.12-0.20231120175607-0b6d5135bbc5/go.mod h1:HPGbmjaaKzGGOL8B0Rksc9+asBEAjnmPkJ2t1Hfl+uI= +github.com/skycoin/dmsg v1.3.12-0.20231130180155-65d707cd1b01 h1:dZD0r4a6pyCpyMYzQxdatmmTScpNj0gYlPQghKfBKes= +github.com/skycoin/dmsg v1.3.12-0.20231130180155-65d707cd1b01/go.mod h1:HPGbmjaaKzGGOL8B0Rksc9+asBEAjnmPkJ2t1Hfl+uI= github.com/skycoin/noise v0.0.0-20180327030543-2492fe189ae6 h1:1Nc5EBY6pjfw1kwW0duwyG+7WliWz5u9kgk1h5MnLuA= github.com/skycoin/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:UXghlricA7J3aRD/k7p/zBObQfmBawwCxIVPVjz2Q3o= github.com/skycoin/skycoin v0.27.1 h1:HatxsRwVSPaV4qxH6290xPBmkH/HgiuAoY2qC+e8C9I= diff --git a/vendor/github.com/skycoin/dmsg/pkg/dmsg/client.go b/vendor/github.com/skycoin/dmsg/pkg/dmsg/client.go index e23860f0..27e86e1c 100644 --- a/vendor/github.com/skycoin/dmsg/pkg/dmsg/client.go +++ b/vendor/github.com/skycoin/dmsg/pkg/dmsg/client.go @@ -472,6 +472,16 @@ func (ce *Client) AllEntries(ctx context.Context) (entries []string, err error) return entries, err } +// ConnectedServersPK return keys of all connected dmsg servers +func (ce *Client) ConnectedServersPK() []string { + sessions := ce.allClientSessions(ce.porter) + addrs := make([]string, len(sessions)) + for i, s := range sessions { + addrs[i] = s.RemotePK().String() + } + return addrs +} + // ConnectionsSummary associates connected clients, and the servers that connect such clients. // Key: Client PK, Value: Slice of Server PKs type ConnectionsSummary map[cipher.PubKey][]cipher.PubKey diff --git a/vendor/modules.txt b/vendor/modules.txt index 210839f0..5e690974 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -455,7 +455,7 @@ github.com/rs/cors ## explicit; go 1.13 github.com/sirupsen/logrus github.com/sirupsen/logrus/hooks/syslog -# github.com/skycoin/dmsg v1.3.12-0.20231120175607-0b6d5135bbc5 +# github.com/skycoin/dmsg v1.3.12-0.20231130180155-65d707cd1b01 ## explicit; go 1.18 github.com/skycoin/dmsg/internal/servermetrics github.com/skycoin/dmsg/pkg/direct From 82695e737aaacedd7683f2b7d90778e09db3a5f4 Mon Sep 17 00:00:00 2001 From: MohammadReza Palide Date: Thu, 30 Nov 2023 18:30:49 +0000 Subject: [PATCH 3/7] add dmsg_servers to AR health endpoint --- cmd/address-resolver/commands/root.go | 8 ++++++++ pkg/address-resolver/api/api.go | 18 +++++++++++------- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/cmd/address-resolver/commands/root.go b/cmd/address-resolver/commands/root.go index 2d7f4db1..386f5cc4 100644 --- a/cmd/address-resolver/commands/root.go +++ b/cmd/address-resolver/commands/root.go @@ -8,6 +8,7 @@ import ( "log/syslog" "os" "strings" + "time" cc "github.com/ivanpirog/coloredcobra" logrussyslog "github.com/sirupsen/logrus/hooks/syslog" @@ -207,6 +208,13 @@ var RootCmd = &cobra.Command{ defer closeDmsgDC() + go func() { + for { + arAPI.DmsgServers = dmsgDC.ConnectedServersPK() + time.Sleep(time.Second) + } + }() + go dmsghttp.UpdateServers(ctx, dClient, dmsgDisc, dmsgDC, logger) go func() { diff --git a/pkg/address-resolver/api/api.go b/pkg/address-resolver/api/api.go index 5a6d1941..bfade500 100644 --- a/pkg/address-resolver/api/api.go +++ b/pkg/address-resolver/api/api.go @@ -64,14 +64,16 @@ type API struct { closeOnce sync.Once closeC chan struct{} - dmsgAddr string + dmsgAddr string + DmsgServers []string } // HealthCheckResponse is struct of /health endpoint type HealthCheckResponse struct { - BuildInfo *buildinfo.Info `json:"build_info,omitempty"` - StartedAt time.Time `json:"started_at"` - DmsgAddr string `json:"dmsg_address,omitempty"` + BuildInfo *buildinfo.Info `json:"build_info,omitempty"` + StartedAt time.Time `json:"started_at"` + DmsgAddr string `json:"dmsg_address,omitempty"` + DmsgServers []string `json:"dmsg_servers,omitempty"` } // ArData has all the visors that have registered with sudph or stcpr transport @@ -97,6 +99,7 @@ func New(log *logging.Logger, s store.Store, nonceStore httpauth.NonceStore, startedAt: time.Now(), closeC: make(chan struct{}), dmsgAddr: dmsgAddr, + DmsgServers: []string{}, } r := chi.NewRouter() @@ -338,9 +341,10 @@ func (a *API) resolve(w http.ResponseWriter, r *http.Request) { func (a *API) health(w http.ResponseWriter, r *http.Request) { info := buildinfo.Get() a.writeJSON(w, r, http.StatusOK, HealthCheckResponse{ - BuildInfo: info, - StartedAt: a.startedAt, - DmsgAddr: a.dmsgAddr, + BuildInfo: info, + StartedAt: a.startedAt, + DmsgAddr: a.dmsgAddr, + DmsgServers: a.DmsgServers, }) } From 17a7097ec293c74da31b4956325aac0b93243ad3 Mon Sep 17 00:00:00 2001 From: MohammadReza Palide Date: Thu, 30 Nov 2023 18:40:33 +0000 Subject: [PATCH 4/7] add dmsg_servers to RF health endpoint --- cmd/route-finder/commands/root.go | 8 ++++++++ pkg/route-finder/api/api.go | 16 ++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/cmd/route-finder/commands/root.go b/cmd/route-finder/commands/root.go index dbb704a3..9d0b9545 100644 --- a/cmd/route-finder/commands/root.go +++ b/cmd/route-finder/commands/root.go @@ -7,6 +7,7 @@ import ( "log" "log/syslog" "os" + "time" cc "github.com/ivanpirog/coloredcobra" logrussyslog "github.com/sirupsen/logrus/hooks/syslog" @@ -168,6 +169,13 @@ var RootCmd = &cobra.Command{ defer closeDmsgDC() + go func() { + for { + rfAPI.DmsgServers = dmsgDC.ConnectedServersPK() + time.Sleep(time.Second) + } + }() + go dmsghttp.UpdateServers(ctx, dClient, dmsgDisc, dmsgDC, logger) go func() { diff --git a/pkg/route-finder/api/api.go b/pkg/route-finder/api/api.go index 076f2ece..0ade0a80 100644 --- a/pkg/route-finder/api/api.go +++ b/pkg/route-finder/api/api.go @@ -30,13 +30,15 @@ type API struct { store store.Store startedAt time.Time dmsgAddr string + DmsgServers []string } // HealthCheckResponse is struct of /health endpoint type HealthCheckResponse struct { - BuildInfo *buildinfo.Info `json:"build_info,omitempty"` - StartedAt time.Time `json:"started_at"` - DmsgAddr string `json:"dmsg_address,omitempty"` + BuildInfo *buildinfo.Info `json:"build_info,omitempty"` + StartedAt time.Time `json:"started_at"` + DmsgAddr string `json:"dmsg_address,omitempty"` + DmsgServers []string `json:"dmsg_servers,omitempty"` } // New creates a new api @@ -46,6 +48,7 @@ func New(s store.Store, logger logrus.FieldLogger, enableMetrics bool, dmsgAddr store: s, startedAt: time.Now(), dmsgAddr: dmsgAddr, + DmsgServers: []string{}, } r := chi.NewRouter() @@ -156,9 +159,10 @@ func (a *API) getPairedRoutes(w http.ResponseWriter, r *http.Request) { func (a *API) health(w http.ResponseWriter, r *http.Request) { info := buildinfo.Get() a.writeJSON(w, r, http.StatusOK, HealthCheckResponse{ - BuildInfo: info, - StartedAt: a.startedAt, - DmsgAddr: a.dmsgAddr, + BuildInfo: info, + StartedAt: a.startedAt, + DmsgAddr: a.dmsgAddr, + DmsgServers: a.DmsgServers, }) } From d036af80add2746fc05117d806366f92660e48ea Mon Sep 17 00:00:00 2001 From: MohammadReza Palide Date: Thu, 30 Nov 2023 18:42:08 +0000 Subject: [PATCH 5/7] add dmsg_servers to TPD health endpoint --- cmd/transport-discovery/commands/root.go | 8 ++++++++ pkg/transport-discovery/api/api.go | 9 ++++++--- pkg/transport-discovery/api/endpoints.go | 7 ++++--- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/cmd/transport-discovery/commands/root.go b/cmd/transport-discovery/commands/root.go index 26094fe4..6165f557 100644 --- a/cmd/transport-discovery/commands/root.go +++ b/cmd/transport-discovery/commands/root.go @@ -8,6 +8,7 @@ import ( "log/syslog" "os" "strings" + "time" cc "github.com/ivanpirog/coloredcobra" logrussyslog "github.com/sirupsen/logrus/hooks/syslog" @@ -203,6 +204,13 @@ var RootCmd = &cobra.Command{ defer closeDmsgDC() + go func() { + for { + tpdAPI.DmsgServers = dmsgDC.ConnectedServersPK() + time.Sleep(time.Second) + } + }() + go dmsghttp.UpdateServers(ctx, dClient, dmsgDisc, dmsgDC, logger) go func() { diff --git a/pkg/transport-discovery/api/api.go b/pkg/transport-discovery/api/api.go index 665343fc..aeb22591 100644 --- a/pkg/transport-discovery/api/api.go +++ b/pkg/transport-discovery/api/api.go @@ -52,13 +52,15 @@ type API struct { store store.Store startedAt time.Time dmsgAddr string + DmsgServers []string } // HealthCheckResponse is struct of /health endpoint type HealthCheckResponse struct { - BuildInfo *buildinfo.Info `json:"build_info,omitempty"` - StartedAt time.Time `json:"started_at"` - DmsgAddr string `json:"dmsg_address,omitempty"` + BuildInfo *buildinfo.Info `json:"build_info,omitempty"` + StartedAt time.Time `json:"started_at"` + DmsgAddr string `json:"dmsg_address,omitempty"` + DmsgServers []string `json:"dmsg_servers,omitempty"` } // New constructs a new API instance. @@ -74,6 +76,7 @@ func New(log logrus.FieldLogger, s store.Store, nonceStore httpauth.NonceStore, store: s, startedAt: time.Now(), dmsgAddr: dmsgAddr, + DmsgServers: []string{}, } r := chi.NewRouter() diff --git a/pkg/transport-discovery/api/endpoints.go b/pkg/transport-discovery/api/endpoints.go index 9e3704b0..b6432316 100644 --- a/pkg/transport-discovery/api/endpoints.go +++ b/pkg/transport-discovery/api/endpoints.go @@ -205,9 +205,10 @@ func (api *API) deregisterTransport(w http.ResponseWriter, r *http.Request) { func (api *API) health(w http.ResponseWriter, r *http.Request) { info := buildinfo.Get() api.writeJSON(w, r, http.StatusOK, HealthCheckResponse{ - BuildInfo: info, - StartedAt: api.startedAt, - DmsgAddr: api.dmsgAddr, + BuildInfo: info, + StartedAt: api.startedAt, + DmsgAddr: api.dmsgAddr, + DmsgServers: api.DmsgServers, }) } From 0e8c524119194fef6d4633326eceb6f6a80b6154 Mon Sep 17 00:00:00 2001 From: MohammadReza Palide Date: Thu, 30 Nov 2023 20:30:35 +0000 Subject: [PATCH 6/7] update go to 1.21 for dmsg-discovery and dmsg-server on integration env --- docker/images/dmsg-discovery/DockerfileInt | 2 +- docker/images/dmsg-server/DockerfileInt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/images/dmsg-discovery/DockerfileInt b/docker/images/dmsg-discovery/DockerfileInt index 88ab8cbb..27f5e1d5 100644 --- a/docker/images/dmsg-discovery/DockerfileInt +++ b/docker/images/dmsg-discovery/DockerfileInt @@ -1,4 +1,4 @@ -FROM golang:1.19-alpine AS builder +FROM golang:1.21-alpine AS builder ARG CGO_ENABLED=0 ENV CGO_ENABLED=${CGO_ENABLED} \ diff --git a/docker/images/dmsg-server/DockerfileInt b/docker/images/dmsg-server/DockerfileInt index bf36dc5c..b00c6b60 100644 --- a/docker/images/dmsg-server/DockerfileInt +++ b/docker/images/dmsg-server/DockerfileInt @@ -1,4 +1,4 @@ -FROM golang:1.19-alpine AS builder +FROM golang:1.21-alpine AS builder ARG CGO_ENABLED=0 ENV CGO_ENABLED=${CGO_ENABLED} \ From abf4bc6b20e8c4111e3f669744a44641461c74f4 Mon Sep 17 00:00:00 2001 From: MohammadReza Palide Date: Thu, 30 Nov 2023 22:26:33 +0000 Subject: [PATCH 7/7] fix docker-compose.yml file for dmsg-disc address on services for integration test --- docker/docker-compose.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2dd0ffb8..585e6303 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -167,7 +167,7 @@ services: depends_on: - tpd-redis - postgres-db - entrypoint: "/release/transport-discovery --pg-host postgres-db --pg-port 5432 --redis redis://tpd-redis:6379 --sk 85789cd7efcdf35340c8f4cb3e15bccdff57efda49af5bcdddaae57374d25ab8" + entrypoint: "/release/transport-discovery --pg-host postgres-db --pg-port 5432 --redis redis://tpd-redis:6379 --sk 85789cd7efcdf35340c8f4cb3e15bccdff57efda49af5bcdddaae57374d25ab8 --dmsg-disc http://dmsg-discovery:9090" stdin_open: true # docker run -i tty: true # docker run -t @@ -190,7 +190,7 @@ services: - "9092:9092" depends_on: - transport-discovery - entrypoint: "/release/route-finder --pg-host postgres-db --pg-port 5432 --sk 640905f9fd60d6bf1ee88519f61035765d65319d9f3bac6c5f2fc14ea5cf15ca" + entrypoint: "/release/route-finder --pg-host postgres-db --pg-port 5432 --sk 640905f9fd60d6bf1ee88519f61035765d65319d9f3bac6c5f2fc14ea5cf15ca --dmsg-disc http://dmsg-discovery:9090" stdin_open: true # docker run -i tty: true # docker run -t @@ -279,7 +279,7 @@ services: - PG_USER=postgres - PG_PASSWORD=postgres - PG_DATABASE=postgres - entrypoint: "/release/service-discovery --pg-host postgres-db --pg-port 5432 --redis redis://sd-redis:6379 --sk ccc3ae9993ed03bed80b1f9feb747d7c6ca3342a65c3923191b7527c2c00ef0f" + entrypoint: "/release/service-discovery --pg-host postgres-db --pg-port 5432 --redis redis://sd-redis:6379 --sk ccc3ae9993ed03bed80b1f9feb747d7c6ca3342a65c3923191b7527c2c00ef0f --dmsg-disc http://dmsg-discovery:9090" stdin_open: true # docker run -i tty: true # docker run -t @@ -303,7 +303,7 @@ services: - PG_DATABASE=postgres ports: - "9096:9096" - entrypoint: "/release/uptime-tracker --pg-host postgres-db --pg-port 5432 --redis redis://ut-redis:6379 --sk e20681b4fd1eb1904ebfbf8fba79a138bbac481e1c3f77bcfab3a877525fda32" + entrypoint: "/release/uptime-tracker --pg-host postgres-db --pg-port 5432 --redis redis://ut-redis:6379 --sk e20681b4fd1eb1904ebfbf8fba79a138bbac481e1c3f77bcfab3a877525fda32 --dmsg-disc http://dmsg-discovery:9090" stdin_open: true # docker run -i tty: true # docker run -t @@ -322,7 +322,7 @@ services: depends_on: - ar-redis restart: on-failure - entrypoint: "/release/address-resolver --redis redis://ar-redis:6379 --sk 1fa0a7b80438b8d31655b5c69efd57bcf931ac828438ff2925ab36e4abcc426a" + entrypoint: "/release/address-resolver --redis redis://ar-redis:6379 --sk 1fa0a7b80438b8d31655b5c69efd57bcf931ac828438ff2925ab36e4abcc426a --dmsg-disc http://dmsg-discovery:9090" stdin_open: true # docker run -i tty: true # docker run -t