From 79164aeae6be67fb665b21df234515a745ea6059 Mon Sep 17 00:00:00 2001 From: iAmir Date: Wed, 4 Oct 2023 06:30:39 +0330 Subject: [PATCH 1/9] reduce server scraper interval --- internal/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/config/config.go b/internal/config/config.go index 8e31c07ab..3f7722bba 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -27,7 +27,7 @@ type Config struct { DocsSourcesPath string `envconfig:"DOCS_SOURCES_PATH" required:"false" default:"docs/"` DocsIndexPath string `envconfig:"DOCS_INDEX_PATH" required:"false" default:"docs.bleve"` PackagesDB string `envconfig:"PACKAGES_DB" required:"false" default:"data/packages.db"` - ServerScrapeInterval time.Duration `envconfig:"SERVER_SCRAPE_INTERVAL" required:"false" default:"45m"` + ServerScrapeInterval time.Duration `envconfig:"SERVER_SCRAPE_INTERVAL" required:"false" default:"30m"` PackageSearchInterval time.Duration `envconfig:"PACKAGE_SEARCH_INTERVAL" required:"false" default:"24h"` PackageScrapeInterval time.Duration `envconfig:"PACKAGE_SCRAPE_INTERVAL" required:"false" default:"24h"` } From 115aa4422cc552b37127f1b498216c6a647ab30c Mon Sep 17 00:00:00 2001 From: iAmir Date: Wed, 4 Oct 2023 06:31:09 +0330 Subject: [PATCH 2/9] add config.CachedServers and env vars for it --- fly.toml | 9 +++++---- internal/config/config.go | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/fly.toml b/fly.toml index 73eb511a7..9bfe98d8a 100644 --- a/fly.toml +++ b/fly.toml @@ -11,10 +11,11 @@ processes = [] auto_rollback = true [env] - LOG_LEVEL = "debug" - PACKAGES_DB = "/data/packages.db" - PRODUCTION = "true" - PUBLIC_WEB_ADDRESS = "https://www.open.mp" + LOG_LEVEL = "debug" + PACKAGES_DB = "/data/packages.db" + CACHED_SERVERS_FILE = "/data/cachedServers.db" + PRODUCTION = "true" + PUBLIC_WEB_ADDRESS = "https://www.open.mp" [mounts] destination = "/data" diff --git a/internal/config/config.go b/internal/config/config.go index 3f7722bba..83eee9227 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -27,6 +27,7 @@ type Config struct { DocsSourcesPath string `envconfig:"DOCS_SOURCES_PATH" required:"false" default:"docs/"` DocsIndexPath string `envconfig:"DOCS_INDEX_PATH" required:"false" default:"docs.bleve"` PackagesDB string `envconfig:"PACKAGES_DB" required:"false" default:"data/packages.db"` + CachedServers string `envconfig:"CACHED_SERVERS_FILE" required:"false" default:"data/cachedServers.json"` ServerScrapeInterval time.Duration `envconfig:"SERVER_SCRAPE_INTERVAL" required:"false" default:"30m"` PackageSearchInterval time.Duration `envconfig:"PACKAGE_SEARCH_INTERVAL" required:"false" default:"24h"` PackageScrapeInterval time.Duration `envconfig:"PACKAGE_SCRAPE_INTERVAL" required:"false" default:"24h"` From 0d733460d56e9975933c52e3b315da3979812f35 Mon Sep 17 00:00:00 2001 From: iAmir Date: Wed, 4 Oct 2023 06:31:42 +0330 Subject: [PATCH 3/9] save servers data into cache file on each scrape run --- app/services/serverworker/serverworker.go | 31 ++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/app/services/serverworker/serverworker.go b/app/services/serverworker/serverworker.go index 3f4889953..51471b922 100644 --- a/app/services/serverworker/serverworker.go +++ b/app/services/serverworker/serverworker.go @@ -2,6 +2,8 @@ package serverworker import ( "context" + "encoding/json" + "os" "time" "github.com/prometheus/client_golang/prometheus" @@ -34,8 +36,9 @@ var ( ) type Worker struct { - db server.Repository - sc scraper.Scraper + db server.Repository + sc scraper.Scraper + cfg config.Config } func Build() fx.Option { @@ -46,7 +49,7 @@ func Build() fx.Option { ), fx.Invoke(func(lc fx.Lifecycle, db server.Repository, sc scraper.Scraper, cfg config.Config) *Worker { - w := &Worker{db, sc} + w := &Worker{db, sc, cfg} lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { @@ -114,6 +117,28 @@ func (w *Worker) Run(ctx context.Context, window time.Duration) error { continue } + // Let's save all servers info our cache file to be used in our API data processing instead of DB + jsonData, err := json.Marshal(all) + if err != nil { + zap.L().Error("There was an error converting native array of servers to JSON data", + zap.Error(err)) + continue + } + + cacheFile, err := os.Create(w.cfg.CachedServers) + if err != nil { + zap.L().Error("Could not create server cache file", + zap.Error(err)) + continue + } + + _, err = cacheFile.Write(jsonData) + if err != nil { + zap.L().Error("There was an error writing collected servers into cache file", + zap.Error(err)) + continue + } + active := 0 inactive := 0 for _, s := range all { From 5e3f8abf345b5ce14976df0cdce8d1378a17ba92 Mon Sep 17 00:00:00 2001 From: iAmir Date: Wed, 4 Oct 2023 06:33:54 +0330 Subject: [PATCH 4/9] remove old cache system usage --- app/transports/api/servers/api.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/transports/api/servers/api.go b/app/transports/api/servers/api.go index 177eb7e58..a0a540e78 100644 --- a/app/transports/api/servers/api.go +++ b/app/transports/api/servers/api.go @@ -36,9 +36,7 @@ func Build() fx.Option { s *service, ) { rtr := chi.NewRouter() - r.With( - cacheClient.Middleware, - ).Mount("/servers", rtr) + r.Mount("/servers", rtr) // TODO: Remove this at some point. r.Mount("/server", rtr) From 39f00d62846d3d12531f91dfd874c82f621e2d97 Mon Sep 17 00:00:00 2001 From: iAmir Date: Wed, 4 Oct 2023 06:34:26 +0330 Subject: [PATCH 5/9] add GetAllCached & GetByAddressCached --- app/resources/server/db.go | 38 ++++++++++++++++++++++++++++-- app/resources/server/repository.go | 2 ++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/app/resources/server/db.go b/app/resources/server/db.go index 898c93609..4195e1db3 100644 --- a/app/resources/server/db.go +++ b/app/resources/server/db.go @@ -2,10 +2,13 @@ package server import ( "context" + "encoding/json" + "os" "time" "github.com/pkg/errors" + "github.com/openmultiplayer/web/internal/config" "github.com/openmultiplayer/web/internal/db" ) @@ -13,10 +16,11 @@ var _ Repository = &DB{} type DB struct { client *db.PrismaClient + cfg config.Config } -func New(client *db.PrismaClient) Repository { - return &DB{client} +func New(client *db.PrismaClient, cfg config.Config) Repository { + return &DB{client, cfg} } func (s *DB) Upsert(ctx context.Context, e All) error { @@ -164,3 +168,33 @@ func (s *DB) SetDeleted(ctx context.Context, ip string, at *time.Time) (*All, er } return dbToAPI(*result), err } + +func (s *DB) GetAllCached(updatedSince time.Duration) ([]All, error) { + result := []All{} + dat, err := os.ReadFile(s.cfg.CachedServers) + if err != nil { + return nil, err + } + + err = json.Unmarshal(dat, &result) + if err != nil { + return nil, err + } + + return result, nil +} + +func (s *DB) GetByAddressCached(address string) (*All, error) { + result, err := s.GetAllCached(-120 * time.Hour) + if err != nil { + return nil, err + } + + for _, n := range result { + if address == n.IP { + return &n, nil + } + } + + return nil, errors.New("server_not_found") +} diff --git a/app/resources/server/repository.go b/app/resources/server/repository.go index becf40b46..57df20562 100644 --- a/app/resources/server/repository.go +++ b/app/resources/server/repository.go @@ -14,4 +14,6 @@ type Repository interface { GetServersToQuery(context.Context, time.Duration) ([]string, error) GetAll(context.Context, time.Duration) ([]All, error) SetDeleted(context.Context, string, *time.Time) (*All, error) + GetAllCached(time.Duration) ([]All, error) + GetByAddressCached(string) (*All, error) } From 6ff89ae9da3c6fccb42b40d63f7e91ae3c4459fb Mon Sep 17 00:00:00 2001 From: iAmir Date: Wed, 4 Oct 2023 06:34:50 +0330 Subject: [PATCH 6/9] use cache functions for server APIs --- app/transports/api/servers/h_get.go | 7 ++++++- app/transports/api/servers/h_list.go | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/app/transports/api/servers/h_get.go b/app/transports/api/servers/h_get.go index 21feb7f05..84b667987 100644 --- a/app/transports/api/servers/h_get.go +++ b/app/transports/api/servers/h_get.go @@ -11,7 +11,12 @@ import ( ) func (s *service) get(w http.ResponseWriter, r *http.Request) { - result, err := s.storer.GetByAddress(r.Context(), chi.URLParam(r, "address")) + // This used to be for getting servers from database directly + // result, err := s.storer.GetByAddress(r.Context(), chi.URLParam(r, "address")) + + // Let's use cached servers, instead of getting them directly from database + // This way we can save a lot DB processing + result, err := s.storer.GetByAddressCached(chi.URLParam(r, "address")) if err != nil { if errors.Is(err, db.ErrNotFound) { web.StatusNotFound(w, err) diff --git a/app/transports/api/servers/h_list.go b/app/transports/api/servers/h_list.go index de3c55304..8deb8b1a2 100644 --- a/app/transports/api/servers/h_list.go +++ b/app/transports/api/servers/h_list.go @@ -18,7 +18,12 @@ func (s *service) list(w http.ResponseWriter, r *http.Request) { since = 12 } - list, err := s.storer.GetAll(r.Context(), time.Duration(-since)*time.Hour) + // This used to be for getting servers from database directly + // list, err := s.storer.GetAll(r.Context(), time.Duration(-since)*time.Hour) + + // Let's use cached servers, instead of getting them directly from database + // This way we can save a lot DB processing + list, err := s.storer.GetAllCached(time.Duration(-since) * time.Hour) if err != nil { web.StatusInternalServerError(w, errors.Wrap(err, "failed to get list of servers")) return From 35797b1c868830a8c97b86f371ab6bb49f62b550 Mon Sep 17 00:00:00 2001 From: iAmir Date: Wed, 4 Oct 2023 06:35:32 +0330 Subject: [PATCH 7/9] new api, able to get full data, not just essential --- app/transports/api/servers/api.go | 1 + app/transports/api/servers/h_fulllist.go | 32 ++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 app/transports/api/servers/h_fulllist.go diff --git a/app/transports/api/servers/api.go b/app/transports/api/servers/api.go index a0a540e78..22bbc6af1 100644 --- a/app/transports/api/servers/api.go +++ b/app/transports/api/servers/api.go @@ -58,6 +58,7 @@ func Build() fx.Option { rtr.Get("/{address}", s.get) rtr.Get("/", s.list) + rtr.Get("/full", s.fulllist) rtr.Post("/", s.add) rtr. With(authentication.MustBeAuthenticated). diff --git a/app/transports/api/servers/h_fulllist.go b/app/transports/api/servers/h_fulllist.go new file mode 100644 index 000000000..210229733 --- /dev/null +++ b/app/transports/api/servers/h_fulllist.go @@ -0,0 +1,32 @@ +package servers + +import ( + "net/http" + "strconv" + "time" + + "github.com/pkg/errors" + + "github.com/openmultiplayer/web/internal/web" +) + +func (s *service) fulllist(w http.ResponseWriter, r *http.Request) { + queries := r.URL.Query() + since, err := strconv.Atoi(queries.Get("activeSince")) + if err != nil { + since = 12 + } + + // This used to be for getting servers from database directly + // list, err := s.storer.GetAll(r.Context(), time.Duration(-since)*time.Hour) + + // Let's use cached servers, instead of getting them directly from database + // This way we can save a lot DB processing + list, err := s.storer.GetAllCached(time.Duration(-since) * time.Hour) + if err != nil { + web.StatusInternalServerError(w, errors.Wrap(err, "failed to get list of servers")) + return + } + + web.Write(w, list) +} From 79a650d765497999e435c17deb2f7687b91e2fa5 Mon Sep 17 00:00:00 2001 From: iAmir Date: Wed, 4 Oct 2023 06:43:38 +0330 Subject: [PATCH 8/9] use a way faster json implementation --- app/resources/server/db.go | 2 +- app/services/serverworker/serverworker.go | 2 +- go.mod | 1 + go.sum | 2 ++ 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/resources/server/db.go b/app/resources/server/db.go index 4195e1db3..fce8f9bde 100644 --- a/app/resources/server/db.go +++ b/app/resources/server/db.go @@ -2,10 +2,10 @@ package server import ( "context" - "encoding/json" "os" "time" + "github.com/goccy/go-json" "github.com/pkg/errors" "github.com/openmultiplayer/web/internal/config" diff --git a/app/services/serverworker/serverworker.go b/app/services/serverworker/serverworker.go index 51471b922..20dbdbe3e 100644 --- a/app/services/serverworker/serverworker.go +++ b/app/services/serverworker/serverworker.go @@ -2,10 +2,10 @@ package serverworker import ( "context" - "encoding/json" "os" "time" + "github.com/goccy/go-json" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "go.uber.org/fx" diff --git a/go.mod b/go.mod index fdb812199..b87253516 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/forPelevin/gomoji v0.0.0-20210718160015-5fcf0e405128 github.com/go-chi/chi v4.1.2+incompatible github.com/go-chi/cors v1.1.1 + github.com/goccy/go-json v0.10.2 // indirect github.com/gomarkdown/markdown v0.0.0-20211105120026-16f708f914c3 github.com/google/go-github v0.0.0-20180819205025-d7732128a00e github.com/google/go-github/v28 v28.1.1 diff --git a/go.sum b/go.sum index a3858a81b..d5cfeaae8 100644 --- a/go.sum +++ b/go.sum @@ -283,6 +283,8 @@ github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= From 747d401d6f26a85745c0d1cfec8140daf6ab8e53 Mon Sep 17 00:00:00 2001 From: iAmir Date: Wed, 4 Oct 2023 06:51:37 +0330 Subject: [PATCH 9/9] bring our old cache system back, to be used along side new one --- app/transports/api/servers/api.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/transports/api/servers/api.go b/app/transports/api/servers/api.go index 22bbc6af1..0afbfcdb3 100644 --- a/app/transports/api/servers/api.go +++ b/app/transports/api/servers/api.go @@ -36,7 +36,9 @@ func Build() fx.Option { s *service, ) { rtr := chi.NewRouter() - r.Mount("/servers", rtr) + r.With( + cacheClient.Middleware, + ).Mount("/servers", rtr) // TODO: Remove this at some point. r.Mount("/server", rtr)