Merge pull request #773 from openmultiplayer/amir/file-cached-servers

Cached servers using json file

app/resources/server/db.go

@@ -2,21 +2,25 @@ package server
 
 import (
  "context"
+ "os"
  "time"
 
+ "github.com/goccy/go-json"
  "github.com/pkg/errors"
 
+ "github.com/openmultiplayer/web/internal/config"
  "github.com/openmultiplayer/web/internal/db"
 )
 
 var _ Repository = &DB{}
 
 type DB struct {
  client *db.PrismaClient
+ cfg config.Config
 }
 
-func New(client *db.PrismaClient) Repository {
- return &DB{client}
+func New(client *db.PrismaClient, cfg config.Config) Repository {
+ return &DB{client, cfg}
 }
 
 func (s *DB) Upsert(ctx context.Context, e All) error {
@@ -164,3 +168,33 @@ func (s *DB) SetDeleted(ctx context.Context, ip string, at *time.Time) (*All, er
  }
  return dbToAPI(*result), err
 }
+
+func (s *DB) GetAllCached(updatedSince time.Duration) ([]All, error) {
+ result := []All{}
+ dat, err := os.ReadFile(s.cfg.CachedServers)
+ if err != nil {
+ return nil, err
+ }
+
+ err = json.Unmarshal(dat, &result)
+ if err != nil {
+ return nil, err
+ }
+
+ return result, nil
+}
+
+func (s *DB) GetByAddressCached(address string) (*All, error) {
+ result, err := s.GetAllCached(-120 * time.Hour)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, n := range result {
+ if address == n.IP {
+ return &n, nil
+ }
+ }
+
+ return nil, errors.New("server_not_found")
+}

app/resources/server/repository.go

@@ -14,4 +14,6 @@ type Repository interface {
  GetServersToQuery(context.Context, time.Duration) ([]string, error)
  GetAll(context.Context, time.Duration) ([]All, error)
  SetDeleted(context.Context, string, *time.Time) (*All, error)
+ GetAllCached(time.Duration) ([]All, error)
+ GetByAddressCached(string) (*All, error)
 }

app/services/serverworker/serverworker.go

@@ -2,8 +2,10 @@ package serverworker
 
 import (
  "context"
+ "os"
  "time"
 
+ "github.com/goccy/go-json"
  "github.com/prometheus/client_golang/prometheus"
  "github.com/prometheus/client_golang/prometheus/promauto"
  "go.uber.org/fx"
@@ -34,8 +36,9 @@ var (
 )
 
 type Worker struct {
- db server.Repository
- sc scraper.Scraper
+ db server.Repository
+ sc scraper.Scraper
+ cfg config.Config
 }
 
 func Build() fx.Option {
@@ -46,7 +49,7 @@ func Build() fx.Option {
  ),
 
  fx.Invoke(func(lc fx.Lifecycle, db server.Repository, sc scraper.Scraper, cfg config.Config) *Worker {
- w := &Worker{db, sc}
+ w := &Worker{db, sc, cfg}
 
  lc.Append(fx.Hook{
  OnStart: func(ctx context.Context) error {
@@ -114,6 +117,28 @@ func (w *Worker) Run(ctx context.Context, window time.Duration) error {
  continue
  }
 
+ // Let's save all servers info our cache file to be used in our API data processing instead of DB
+ jsonData, err := json.Marshal(all)
+ if err != nil {
+ zap.L().Error("There was an error converting native array of servers to JSON data",
+ zap.Error(err))
+ continue
+ }
+
+ cacheFile, err := os.Create(w.cfg.CachedServers)
+ if err != nil {
+ zap.L().Error("Could not create server cache file",
+ zap.Error(err))
+ continue
+ }
+
+ _, err = cacheFile.Write(jsonData)
+ if err != nil {
+ zap.L().Error("There was an error writing collected servers into cache file",
+ zap.Error(err))
+ continue
+ }
+
  active := 0
  inactive := 0
  for _, s := range all {

app/transports/api/servers/api.go

@@ -60,6 +60,7 @@ func Build() fx.Option {
 
  rtr.Get("/{address}", s.get)
  rtr.Get("/", s.list)
+ rtr.Get("/full", s.fulllist)
  rtr.Post("/", s.add)
  rtr.
  With(authentication.MustBeAuthenticated).

app/transports/api/servers/h_fulllist.go

@@ -0,0 +1,32 @@
+package servers
+
+import (
+ "net/http"
+ "strconv"
+ "time"
+
+ "github.com/pkg/errors"
+
+ "github.com/openmultiplayer/web/internal/web"
+)
+
+func (s *service) fulllist(w http.ResponseWriter, r *http.Request) {
+ queries := r.URL.Query()
+ since, err := strconv.Atoi(queries.Get("activeSince"))
+ if err != nil {
+ since = 12
+ }
+
+ // This used to be for getting servers from database directly
+ // list, err := s.storer.GetAll(r.Context(), time.Duration(-since)*time.Hour)
+
+ // Let's use cached servers, instead of getting them directly from database
+ // This way we can save a lot DB processing
+ list, err := s.storer.GetAllCached(time.Duration(-since) * time.Hour)
+ if err != nil {
+ web.StatusInternalServerError(w, errors.Wrap(err, "failed to get list of servers"))
+ return
+ }
+
+ web.Write(w, list)
+}

app/transports/api/servers/h_get.go

@@ -11,7 +11,12 @@ import (
 )
 
 func (s *service) get(w http.ResponseWriter, r *http.Request) {
- result, err := s.storer.GetByAddress(r.Context(), chi.URLParam(r, "address"))
+ // This used to be for getting servers from database directly
+ // result, err := s.storer.GetByAddress(r.Context(), chi.URLParam(r, "address"))
+
+ // Let's use cached servers, instead of getting them directly from database
+ // This way we can save a lot DB processing
+ result, err := s.storer.GetByAddressCached(chi.URLParam(r, "address"))
  if err != nil {
  if errors.Is(err, db.ErrNotFound) {
  web.StatusNotFound(w, err)

app/transports/api/servers/h_list.go

@@ -18,7 +18,12 @@ func (s *service) list(w http.ResponseWriter, r *http.Request) {
  since = 12
  }
 
- list, err := s.storer.GetAll(r.Context(), time.Duration(-since)*time.Hour)
+ // This used to be for getting servers from database directly
+ // list, err := s.storer.GetAll(r.Context(), time.Duration(-since)*time.Hour)
+
+ // Let's use cached servers, instead of getting them directly from database
+ // This way we can save a lot DB processing
+ list, err := s.storer.GetAllCached(time.Duration(-since) * time.Hour)
  if err != nil {
  web.StatusInternalServerError(w, errors.Wrap(err, "failed to get list of servers"))
  return

fly.toml

@@ -11,10 +11,11 @@ processes = []
  auto_rollback = true
 
 [env]
- LOG_LEVEL = "debug"
- PACKAGES_DB = "/data/packages.db"
- PRODUCTION = "true"
- PUBLIC_WEB_ADDRESS = "https://www.open.mp"
+ LOG_LEVEL = "debug"
+ PACKAGES_DB = "/data/packages.db"
+ CACHED_SERVERS_FILE = "/data/cachedServers.db"
+ PRODUCTION = "true"
+ PUBLIC_WEB_ADDRESS = "https://www.open.mp"
 
 [mounts]
  destination = "/data"

go.mod

@@ -17,6 +17,7 @@ require (
  github.com/forPelevin/gomoji v0.0.0-20210718160015-5fcf0e405128
  github.com/go-chi/chi v4.1.2+incompatible
  github.com/go-chi/cors v1.1.1
+ github.com/goccy/go-json v0.10.2 // indirect
  github.com/gomarkdown/markdown v0.0.0-20211105120026-16f708f914c3
  github.com/google/go-github v0.0.0-20180819205025-d7732128a00e
  github.com/google/go-github/v28 v28.1.1

go.sum

@@ -283,6 +283,8 @@ github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
 github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
 github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
 github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
+github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
+github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=

internal/config/config.go

@@ -27,7 +27,8 @@ type Config struct {
  DocsSourcesPath string `envconfig:"DOCS_SOURCES_PATH" required:"false" default:"docs/"`
  DocsIndexPath string `envconfig:"DOCS_INDEX_PATH" required:"false" default:"docs.bleve"`
  PackagesDB string `envconfig:"PACKAGES_DB" required:"false" default:"data/packages.db"`
- ServerScrapeInterval time.Duration `envconfig:"SERVER_SCRAPE_INTERVAL" required:"false" default:"45m"`
+ CachedServers string `envconfig:"CACHED_SERVERS_FILE" required:"false" default:"data/cachedServers.json"`
+ ServerScrapeInterval time.Duration `envconfig:"SERVER_SCRAPE_INTERVAL" required:"false" default:"30m"`
  PackageSearchInterval time.Duration `envconfig:"PACKAGE_SEARCH_INTERVAL" required:"false" default:"24h"`
  PackageScrapeInterval time.Duration `envconfig:"PACKAGE_SCRAPE_INTERVAL" required:"false" default:"24h"`
 }