Cache all package manifest on startup (#103)

So far in each request to search or categories all manifest had to be reread from disk each time. As the packages do not change during the runtime of the service, this is not necessary and all can be cached on startup of the service and kept in memory. Even if there are one day 10000 packages and each manifest is 1KB, this would only consume ~10MB of memory (currently manifest are only about 250 Bytes). Using 10MB of memory instead of 10000 file read on each request seems worth it.

The code was also modified to in general pass a copy of the packages instead of using a reference. This makes sure even if a request modifies the packages locally, the global package list is not modified.

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Add validation check that Kibana min/max are valid semver versions. [#99](https://github.com/elastic/integrations-registry/pull/99)
 * Adding Cache-Control max-age headers to all http responses set to 1h. [#101](https://github.com/elastic/integrations-registry/pull/101)
 * Validate packages to guarantee only predefined categories can be used. [#100](https://github.com/elastic/integrations-registry/pull/100)
+* Cache all manifest on service startup for resource optimisation. [#103](https://github.com/elastic/integrations-registry/pull/103)
 
 ### Changed
 

categories.go

@@ -1,3 +1,7 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package main
 
 import (
@@ -9,8 +13,6 @@ import (
  "github.com/elastic/integrations-registry/util"
 )
 
-
-
 type Category struct {
  Id string `yaml:"id" json:"id"`
  Title string `yaml:"title" json:"title"`
@@ -22,19 +24,14 @@ func categoriesHandler() func(w http.ResponseWriter, r *http.Request) {
  return func(w http.ResponseWriter, r *http.Request) {
  cacheHeaders(w)
 
- packagePaths, err := util.GetPackagePaths(packagesBasePath)
+ packages, err := util.GetPackages(packagesBasePath)
  if err != nil {
  notFound(w, err)
  return
  }
-
- packageList := map[string]*util.Package{}
+ packageList := map[string]util.Package{}
  // Get unique list of newest packages
- for _, i := range packagePaths {
- p, err := util.NewPackage(packagesBasePath, i)
- if err != nil {
- return
- }
+ for _, p := range packages {
 
  // Check if the version exists and if it should be added or not.
  if pp, ok := packageList[p.Name]; ok {

magefile.go

@@ -90,21 +90,21 @@ func BuildIntegrationPackages() error {
  packagesBasePath = publicDir + "/" + packageDir + "/"
  }
 
- packagePaths, err := util.GetPackagePaths(packagesBasePath)
+ packages, err := util.GetPackages(packagesBasePath)
  if err != nil {
  return err
  }
 
- for _, path := range packagePaths {
- err = buildPackage(packagesBasePath, path)
+ for _, p := range packages {
+ err = buildPackage(packagesBasePath, p)
  if err != nil {
  return err
  }
  }
  return nil
 }
 
-func buildPackage(packagesBasePath, path string) error {
+func buildPackage(packagesBasePath string, p util.Package) error {
 
  // Change path to simplify tar command
  currentPath, err := os.Getwd()
@@ -117,29 +117,23 @@ func buildPackage(packagesBasePath, path string) error {
  }
  defer os.Chdir(currentPath)
 
- err = sh.RunV("tar", "cvzf", path+".tar.gz", filepath.Base(path)+"/")
+ err = sh.RunV("tar", "cvzf", p.GetPath()+".tar.gz", filepath.Base(p.GetPath())+"/")
  if err != nil {
- return err
- }
-
- // Build package endpoint
- p, err := util.NewPackage(".", path)
- if err != nil {
- return fmt.Errorf("Error creating package: %s: %s", path, err)
+ return fmt.Errorf("Error creating package: %s: %s", p.GetPath(), err)
  }
 
  // Checks if the package is valid
  err = p.Validate()
  if err != nil {
- return fmt.Errorf("Invalid package %s-%s: %s", p.Name, p.Version, err)
+ return fmt.Errorf("Invalid package: %s: %s", p.GetPath(), err)
  }
 
- err = p.LoadAssets(path)
+ err = p.LoadAssets(p.GetPath())
  if err != nil {
  return err
  }
 
- err = writeJsonFile(p, path+"/index.json")
+ err = writeJsonFile(p, p.GetPath()+"/index.json")
  if err != nil {
  return err
  }

main.go

@@ -14,6 +14,8 @@ import (
  "strconv"
  "syscall"
 
+ "github.com/elastic/integrations-registry/util"
+
  ucfgYAML "github.com/elastic/go-ucfg/yaml"
 
  "github.com/gorilla/mux"
@@ -46,6 +48,14 @@ func main() {
  }
  packagesBasePath = config.PackagesPath
 
+ // Prefill the package cache
+ packages, err := util.GetPackages(packagesBasePath)
+ if err != nil {
+ log.Print(err)
+ os.Exit(1)
+ }
+ log.Printf("%v package manifests loaded into memory.\n", len(packages))
+
  server := &http.Server{Addr: address, Handler: getRouter()}
 
  go func() {

search.go

@@ -1,3 +1,7 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package main
 
 import (
@@ -47,21 +51,15 @@ func searchHandler() func(w http.ResponseWriter, r *http.Request) {
  }
  }
 
- packagePaths, err := util.GetPackagePaths(packagesBasePath)
+ packages, err := util.GetPackages(packagesBasePath)
  if err != nil {
- notFound(w, err)
+ notFound(w, fmt.Errorf("problem fetching packages: %s", err))
  return
  }
-
- packagesList := map[string]map[string]*util.Package{}
+ packagesList := map[string]map[string]util.Package{}
 
  // Checks that only the most recent version of an integration is added to the list
- for _, path := range packagePaths {
- p, err := util.NewPackage(packagesBasePath, path)
- if err != nil {
- notFound(w, err)
- return
- }
+ for _, p := range packages {
 
  // Filter by category first as this could heavily reduce the number of packages
  // It must happen before the version filtering as there only the newest version
@@ -100,7 +98,7 @@ func searchHandler() func(w http.ResponseWriter, r *http.Request) {
  }
 
  if _, ok := packagesList[p.Name]; !ok {
- packagesList[p.Name] = map[string]*util.Package{}
+ packagesList[p.Name] = map[string]util.Package{}
  }
  packagesList[p.Name][p.Version] = p
  }
@@ -116,7 +114,7 @@ func searchHandler() func(w http.ResponseWriter, r *http.Request) {
  }
 }
 
-func getPackageOutput(packagesList map[string]map[string]*util.Package) ([]byte, error) {
+func getPackageOutput(packagesList map[string]map[string]util.Package) ([]byte, error) {
 
  separator := "@"
  // Packages need to be sorted to be always outputted in the same order

util/package.go

@@ -1,3 +1,7 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package util
 
 import (
@@ -8,10 +12,9 @@ import (
  "strings"
 
  "github.com/pkg/errors"
+ yaml "gopkg.in/yaml.v2"
 
  "github.com/blang/semver"
-
- "gopkg.in/yaml.v2"
 )
 
 const defaultType = "integration"
@@ -137,7 +140,7 @@ func (p *Package) HasKibanaVersion(version *semver.Version) bool {
  return true
 }
 
-func (p *Package) IsNewer(pp *Package) bool {
+func (p *Package) IsNewer(pp Package) bool {
  return p.versionSemVer.GT(pp.versionSemVer)
 }
 
@@ -230,3 +233,7 @@ func (p *Package) Validate() error {
 
  return nil
 }
+
+func (p *Package) GetPath() string {
+ return p.Name + "-" + p.Version
+}

util/package_test.go

@@ -1,3 +1,7 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package util
 
 import (

util/packages.go

@@ -1,9 +1,41 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package util
 
-import "io/ioutil"
+import (
+ "io/ioutil"
+)
+
+var packageList []Package
+
+// GetPackages returns a slice with all existing packages.
+// The list is stored in memory and on the second request directly
+// served from memory. This assumes chnages to packages only happen on restart.
+// Caching the packages request many file reads every time this method is called.
+func GetPackages(packagesBasePath string) ([]Package, error) {
+ if packageList != nil {
+ return packageList, nil
+ }
+
+ packagePaths, err := getPackagePaths(packagesBasePath)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, i := range packagePaths {
+ p, err := NewPackage(packagesBasePath, i)
+ if err != nil {
+ return nil, err
+ }
+ packageList = append(packageList, *p)
+ }
+ return packageList, nil
+}
 
-// GetPackagePaths returns list of available packages, one for each version.
-func GetPackagePaths(packagesPath string) ([]string, error) {
+// getPackagePaths returns list of available packages, one for each version.
+func getPackagePaths(packagesPath string) ([]string, error) {
 
  files, err := ioutil.ReadDir(packagesPath)
  if err != nil {