Cache all package manifest on startup

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Add validation check that Kibana min/max are valid semver versions. [#99](https://github.com/elastic/integrations-registry/pull/99)
 * Adding Cache-Control max-age headers to all http responses set to 1h. [#101](https://github.com/elastic/integrations-registry/pull/101)
 * Validate packages to guarantee only predefined categories can be used. [#100](https://github.com/elastic/integrations-registry/pull/100)
+* Cache all manifest on service startup for resource optimisation. [#103](https://github.com/elastic/integrations-registry/pull/103)
 
 ### Changed
 

categories.go

@@ -1,3 +1,7 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package main
 
 import (
@@ -9,8 +13,6 @@ import (
  "github.com/elastic/integrations-registry/util"
 )
 
-
-
 type Category struct {
  Id string `yaml:"id" json:"id"`
  Title string `yaml:"title" json:"title"`
@@ -22,19 +24,14 @@ func categoriesHandler() func(w http.ResponseWriter, r *http.Request) {
  return func(w http.ResponseWriter, r *http.Request) {
  cacheHeaders(w)
 
- packagePaths, err := util.GetPackagePaths(packagesBasePath)
+ packages, err := util.GetPackages(packagesBasePath)
  if err != nil {
  notFound(w, err)
  return
  }
-
- packageList := map[string]*util.Package{}
+ packageList := map[string]util.Package{}
  // Get unique list of newest packages
- for _, i := range packagePaths {
- p, err := util.NewPackage(packagesBasePath, i)
- if err != nil {
- return
- }
+ for _, p := range packages {
 
  // Check if the version exists and if it should be added or not.
  if pp, ok := packageList[p.Name]; ok {

magefile.go

@@ -90,21 +90,21 @@ func BuildIntegrationPackages() error {
  packagesBasePath = publicDir + "/" + packageDir + "/"
  }
 
- packagePaths, err := util.GetPackagePaths(packagesBasePath)
+ packages, err := util.GetPackages(packagesBasePath)
  if err != nil {
  return err
  }
 
- for _, path := range packagePaths {
- err = buildPackage(packagesBasePath, path)
+ for _, p := range packages {
+ err = buildPackage(packagesBasePath, p)
  if err != nil {
  return err
  }
  }
  return nil
 }
 
-func buildPackage(packagesBasePath, path string) error {
+func buildPackage(packagesBasePath string, p util.Package) error {
 
  // Change path to simplify tar command
  currentPath, err := os.Getwd()
@@ -117,29 +117,23 @@ func buildPackage(packagesBasePath, path string) error {
  }
  defer os.Chdir(currentPath)
 
- err = sh.RunV("tar", "cvzf", path+".tar.gz", filepath.Base(path)+"/")
+ err = sh.RunV("tar", "cvzf", p.GetPath()+".tar.gz", filepath.Base(p.GetPath())+"/")
  if err != nil {
- return err
- }
-
- // Build package endpoint
- p, err := util.NewPackage(".", path)
- if err != nil {
- return fmt.Errorf("Error creating package: %s: %s", path, err)
+ return fmt.Errorf("Error creating package: %s: %s", p.GetPath(), err)
  }
 
  // Checks if the package is valid
  err = p.Validate()
  if err != nil {
- return fmt.Errorf("Invalid package %s-%s: %s", p.Name, p.Version, err)
+ return fmt.Errorf("Invalid package: %s: %s", p.GetPath(), err)
  }
 
- err = p.LoadAssets(path)
+ err = p.LoadAssets(p.GetPath())
  if err != nil {
  return err
  }
 
- err = writeJsonFile(p, path+"/index.json")
+ err = writeJsonFile(p, p.GetPath()+"/index.json")
  if err != nil {
  return err
  }

main.go

@@ -14,6 +14,8 @@ import (
  "strconv"
  "syscall"
 
+ "github.com/elastic/integrations-registry/util"
+
  ucfgYAML "github.com/elastic/go-ucfg/yaml"
 
  "github.com/gorilla/mux"
@@ -46,6 +48,14 @@ func main() {
  }
  packagesBasePath = config.PackagesPath
 
+ // Prefill the package cache
+ packages, err := util.GetPackages(packagesBasePath)
+ if err != nil {
+ log.Print(err)
+ os.Exit(1)
+ }
+ log.Printf("%v package manifests loaded into memory.\n", len(packages))
+
  server := &http.Server{Addr: address, Handler: getRouter()}
 
  go func() {

search.go

@@ -1,3 +1,7 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package main
 
 import (
@@ -47,21 +51,15 @@ func searchHandler() func(w http.ResponseWriter, r *http.Request) {
  }
  }
 
- packagePaths, err := util.GetPackagePaths(packagesBasePath)
+ packages, err := util.GetPackages(packagesBasePath)
  if err != nil {
- notFound(w, err)
+ notFound(w, fmt.Errorf("problem fetching packages: %s", err))
  return
  }
-
- packagesList := map[string]map[string]*util.Package{}
+ packagesList := map[string]map[string]util.Package{}
 
  // Checks that only the most recent version of an integration is added to the list
- for _, path := range packagePaths {
- p, err := util.NewPackage(packagesBasePath, path)
- if err != nil {
- notFound(w, err)
- return
- }
+ for _, p := range packages {
 
  // Filter by category first as this could heavily reduce the number of packages
  // It must happen before the version filtering as there only the newest version
@@ -100,7 +98,7 @@ func searchHandler() func(w http.ResponseWriter, r *http.Request) {
  }
 
  if _, ok := packagesList[p.Name]; !ok {
- packagesList[p.Name] = map[string]*util.Package{}
+ packagesList[p.Name] = map[string]util.Package{}
  }
  packagesList[p.Name][p.Version] = p
  }
@@ -116,7 +114,7 @@ func searchHandler() func(w http.ResponseWriter, r *http.Request) {
  }
 }
 
-func getPackageOutput(packagesList map[string]map[string]*util.Package) ([]byte, error) {
+func getPackageOutput(packagesList map[string]map[string]util.Package) ([]byte, error) {
 
  separator := "@"
  // Packages need to be sorted to be always outputted in the same order

util/package.go

@@ -1,3 +1,7 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package util
 
 import (
@@ -8,10 +12,9 @@ import (
  "strings"
 
  "github.com/pkg/errors"
+ yaml "gopkg.in/yaml.v2"
 
  "github.com/blang/semver"
-
- "gopkg.in/yaml.v2"
 )
 
 const defaultType = "integration"
@@ -137,7 +140,7 @@ func (p *Package) HasKibanaVersion(version *semver.Version) bool {
  return true
 }
 
-func (p *Package) IsNewer(pp *Package) bool {
+func (p *Package) IsNewer(pp Package) bool {
  return p.versionSemVer.GT(pp.versionSemVer)
 }
 
@@ -230,3 +233,7 @@ func (p *Package) Validate() error {
 
  return nil
 }
+
+func (p *Package) GetPath() string {
+ return p.Name + "-" + p.Version
+}

util/package_test.go

@@ -1,3 +1,7 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package util
 
 import (

util/packages.go

@@ -1,9 +1,41 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package util
 
-import "io/ioutil"
+import (
+ "io/ioutil"
+)
+
+var packageList []Package
+
+// GetPackages returns a slice with all existing packages.
+// The list is stored in memory and on the second request directly
+// served from memory. This assumes chnages to packages only happen on restart.
+// Caching the packages request many file reads every time this method is called.
+func GetPackages(packagesBasePath string) ([]Package, error) {
+ if packageList != nil {
+ return packageList, nil
+ }
+
+ packagePaths, err := getPackagePaths(packagesBasePath)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, i := range packagePaths {
+ p, err := NewPackage(packagesBasePath, i)
+ if err != nil {
+ return nil, err
+ }
+ packageList = append(packageList, *p)
+ }
+ return packageList, nil
+}
 
-// GetPackagePaths returns list of available packages, one for each version.
-func GetPackagePaths(packagesPath string) ([]string, error) {
+// getPackagePaths returns list of available packages, one for each version.
+func getPackagePaths(packagesPath string) ([]string, error) {
 
  files, err := ioutil.ReadDir(packagesPath)
  if err != nil {