From a575c39a920651af0005c47e3c708b265a5b9243 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Thu, 12 Sep 2019 15:45:10 +0200 Subject: [PATCH] Cache all package manifest on startup (#103) So far in each request to search or categories all manifest had to be reread from disk each time. As the packages do not change during the runtime of the service, this is not necessary and all can be cached on startup of the service and kept in memory. Even if there are one day 10000 packages and each manifest is 1KB, this would only consume ~10MB of memory (currently manifest are only about 250 Bytes). Using 10MB of memory instead of 10000 file read on each request seems worth it. The code was also modified to in general pass a copy of the packages instead of using a reference. This makes sure even if a request modifies the packages locally, the global package list is not modified. --- CHANGELOG.md | 1 + categories.go | 17 +++++++---------- magefile.go | 24 +++++++++--------------- main.go | 10 ++++++++++ search.go | 22 ++++++++++------------ util/package.go | 13 ++++++++++--- util/package_test.go | 4 ++++ util/packages.go | 38 +++++++++++++++++++++++++++++++++++--- 8 files changed, 86 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72fe1ad6f..7d10a35e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Add validation check that Kibana min/max are valid semver versions. [#99](https://github.com/elastic/integrations-registry/pull/99) * Adding Cache-Control max-age headers to all http responses set to 1h. [#101](https://github.com/elastic/integrations-registry/pull/101) * Validate packages to guarantee only predefined categories can be used. [#100](https://github.com/elastic/integrations-registry/pull/100) +* Cache all manifest on service startup for resource optimisation. [#103](https://github.com/elastic/integrations-registry/pull/103) ### Changed diff --git a/categories.go b/categories.go index e6ca1aff3..adf09d695 100644 --- a/categories.go +++ b/categories.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package main import ( @@ -9,8 +13,6 @@ import ( "github.com/elastic/integrations-registry/util" ) - - type Category struct { Id string `yaml:"id" json:"id"` Title string `yaml:"title" json:"title"` @@ -22,19 +24,14 @@ func categoriesHandler() func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { cacheHeaders(w) - packagePaths, err := util.GetPackagePaths(packagesBasePath) + packages, err := util.GetPackages(packagesBasePath) if err != nil { notFound(w, err) return } - - packageList := map[string]*util.Package{} + packageList := map[string]util.Package{} // Get unique list of newest packages - for _, i := range packagePaths { - p, err := util.NewPackage(packagesBasePath, i) - if err != nil { - return - } + for _, p := range packages { // Check if the version exists and if it should be added or not. if pp, ok := packageList[p.Name]; ok { diff --git a/magefile.go b/magefile.go index 830d0d8c1..aa21589bf 100644 --- a/magefile.go +++ b/magefile.go @@ -90,13 +90,13 @@ func BuildIntegrationPackages() error { packagesBasePath = publicDir + "/" + packageDir + "/" } - packagePaths, err := util.GetPackagePaths(packagesBasePath) + packages, err := util.GetPackages(packagesBasePath) if err != nil { return err } - for _, path := range packagePaths { - err = buildPackage(packagesBasePath, path) + for _, p := range packages { + err = buildPackage(packagesBasePath, p) if err != nil { return err } @@ -104,7 +104,7 @@ func BuildIntegrationPackages() error { return nil } -func buildPackage(packagesBasePath, path string) error { +func buildPackage(packagesBasePath string, p util.Package) error { // Change path to simplify tar command currentPath, err := os.Getwd() @@ -117,29 +117,23 @@ func buildPackage(packagesBasePath, path string) error { } defer os.Chdir(currentPath) - err = sh.RunV("tar", "cvzf", path+".tar.gz", filepath.Base(path)+"/") + err = sh.RunV("tar", "cvzf", p.GetPath()+".tar.gz", filepath.Base(p.GetPath())+"/") if err != nil { - return err - } - - // Build package endpoint - p, err := util.NewPackage(".", path) - if err != nil { - return fmt.Errorf("Error creating package: %s: %s", path, err) + return fmt.Errorf("Error creating package: %s: %s", p.GetPath(), err) } // Checks if the package is valid err = p.Validate() if err != nil { - return fmt.Errorf("Invalid package %s-%s: %s", p.Name, p.Version, err) + return fmt.Errorf("Invalid package: %s: %s", p.GetPath(), err) } - err = p.LoadAssets(path) + err = p.LoadAssets(p.GetPath()) if err != nil { return err } - err = writeJsonFile(p, path+"/index.json") + err = writeJsonFile(p, p.GetPath()+"/index.json") if err != nil { return err } diff --git a/main.go b/main.go index a882a139d..05d133552 100644 --- a/main.go +++ b/main.go @@ -14,6 +14,8 @@ import ( "strconv" "syscall" + "github.com/elastic/integrations-registry/util" + ucfgYAML "github.com/elastic/go-ucfg/yaml" "github.com/gorilla/mux" @@ -46,6 +48,14 @@ func main() { } packagesBasePath = config.PackagesPath + // Prefill the package cache + packages, err := util.GetPackages(packagesBasePath) + if err != nil { + log.Print(err) + os.Exit(1) + } + log.Printf("%v package manifests loaded into memory.\n", len(packages)) + server := &http.Server{Addr: address, Handler: getRouter()} go func() { diff --git a/search.go b/search.go index 055c8130c..8e1f91cbf 100644 --- a/search.go +++ b/search.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package main import ( @@ -47,21 +51,15 @@ func searchHandler() func(w http.ResponseWriter, r *http.Request) { } } - packagePaths, err := util.GetPackagePaths(packagesBasePath) + packages, err := util.GetPackages(packagesBasePath) if err != nil { - notFound(w, err) + notFound(w, fmt.Errorf("problem fetching packages: %s", err)) return } - - packagesList := map[string]map[string]*util.Package{} + packagesList := map[string]map[string]util.Package{} // Checks that only the most recent version of an integration is added to the list - for _, path := range packagePaths { - p, err := util.NewPackage(packagesBasePath, path) - if err != nil { - notFound(w, err) - return - } + for _, p := range packages { // Filter by category first as this could heavily reduce the number of packages // It must happen before the version filtering as there only the newest version @@ -100,7 +98,7 @@ func searchHandler() func(w http.ResponseWriter, r *http.Request) { } if _, ok := packagesList[p.Name]; !ok { - packagesList[p.Name] = map[string]*util.Package{} + packagesList[p.Name] = map[string]util.Package{} } packagesList[p.Name][p.Version] = p } @@ -116,7 +114,7 @@ func searchHandler() func(w http.ResponseWriter, r *http.Request) { } } -func getPackageOutput(packagesList map[string]map[string]*util.Package) ([]byte, error) { +func getPackageOutput(packagesList map[string]map[string]util.Package) ([]byte, error) { separator := "@" // Packages need to be sorted to be always outputted in the same order diff --git a/util/package.go b/util/package.go index 6722254fd..16c0d7366 100644 --- a/util/package.go +++ b/util/package.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package util import ( @@ -8,10 +12,9 @@ import ( "strings" "github.com/pkg/errors" + yaml "gopkg.in/yaml.v2" "github.com/blang/semver" - - "gopkg.in/yaml.v2" ) const defaultType = "integration" @@ -137,7 +140,7 @@ func (p *Package) HasKibanaVersion(version *semver.Version) bool { return true } -func (p *Package) IsNewer(pp *Package) bool { +func (p *Package) IsNewer(pp Package) bool { return p.versionSemVer.GT(pp.versionSemVer) } @@ -230,3 +233,7 @@ func (p *Package) Validate() error { return nil } + +func (p *Package) GetPath() string { + return p.Name + "-" + p.Version +} diff --git a/util/package_test.go b/util/package_test.go index f271751b2..5b52e34c1 100644 --- a/util/package_test.go +++ b/util/package_test.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package util import ( diff --git a/util/packages.go b/util/packages.go index d6c2c5c5c..35633431b 100644 --- a/util/packages.go +++ b/util/packages.go @@ -1,9 +1,41 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package util -import "io/ioutil" +import ( + "io/ioutil" +) + +var packageList []Package + +// GetPackages returns a slice with all existing packages. +// The list is stored in memory and on the second request directly +// served from memory. This assumes chnages to packages only happen on restart. +// Caching the packages request many file reads every time this method is called. +func GetPackages(packagesBasePath string) ([]Package, error) { + if packageList != nil { + return packageList, nil + } + + packagePaths, err := getPackagePaths(packagesBasePath) + if err != nil { + return nil, err + } + + for _, i := range packagePaths { + p, err := NewPackage(packagesBasePath, i) + if err != nil { + return nil, err + } + packageList = append(packageList, *p) + } + return packageList, nil +} -// GetPackagePaths returns list of available packages, one for each version. -func GetPackagePaths(packagesPath string) ([]string, error) { +// getPackagePaths returns list of available packages, one for each version. +func getPackagePaths(packagesPath string) ([]string, error) { files, err := ioutil.ReadDir(packagesPath) if err != nil {