Implement proxy mode (#860)

* WIP

* Refactor indexer to proxy mode

* Prepare interfaces

* Link

* Bump github.com/prometheus/client_golang from 1.12.2 to 1.13.0 (#861)

Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.12.2 to 1.13.0.
- [Release notes](https://github.com/prometheus/client_golang/releases)
- [Changelog](https://github.com/prometheus/client_golang/blob/main/CHANGELOG.md)
- [Commits](prometheus/client_golang@v1.12.2...v1.13.0)

---
updated-dependencies:
- dependency-name: github.com/prometheus/client_golang
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* Bump go.uber.org/zap from 1.21.0 to 1.22.0 (#862)

Bumps [go.uber.org/zap](https://github.com/uber-go/zap) from 1.21.0 to 1.22.0.
- [Release notes](https://github.com/uber-go/zap/releases)
- [Changelog](https://github.com/uber-go/zap/blob/master/CHANGELOG.md)
- [Commits](uber-go/zap@v1.21.0...v1.22.0)

---
updated-dependencies:
- dependency-name: go.uber.org/zap
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* Update favicon with the new EPR logo (#858)

Update favicon file with the new Elastic Package Registry logo.
A new "img" folder has also been added to include other
available formats of this new logo (png, svg and ico).

* Bump cloud.google.com/go/storage from 1.24.0 to 1.25.0 (#863)

Bumps [cloud.google.com/go/storage](https://github.com/googleapis/google-cloud-go) from 1.24.0 to 1.25.0.
- [Release notes](https://github.com/googleapis/google-cloud-go/releases)
- [Changelog](https://github.com/googleapis/google-cloud-go/blob/main/CHANGES.md)
- [Commits](googleapis/google-cloud-go@pubsub/v1.24.0...spanner/v1.25.0)

---
updated-dependencies:
- dependency-name: cloud.google.com/go/storage
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* ci: transform git tag with `v` format (#865)

* Bump github.com/fsouza/fake-gcs-server from 1.38.3 to 1.38.4 (#866)

Bumps [github.com/fsouza/fake-gcs-server](https://github.com/fsouza/fake-gcs-server) from 1.38.3 to 1.38.4.
- [Release notes](https://github.com/fsouza/fake-gcs-server/releases)
- [Commits](fsouza/fake-gcs-server@v1.38.3...v1.38.4)

---
updated-dependencies:
- dependency-name: github.com/fsouza/fake-gcs-server
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* Use proxymode in main

* Search

* WIP: before resolver

* Resolver

* Basic resolver

* Fix: mage check

* Not needed

* Fix: imports

* Add CHANGELOG

* Categories

* Address PR comments

* Use consts

* More fixes

* Use snapshot registry

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Co-authored-by: Victor Martinez <victormartinezrubio@gmail.com>

CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 * Update favicon to be the Elastic Package Registry logo. [#858](https://github.com/elastic/package-registry/pull/858)
+* Implement proxy mode. [#860](https://github.com/elastic/package-registry/pull/860)
 
 ### Deprecated
 

artifacts.go

@@ -14,6 +14,7 @@ import (
  "go.uber.org/zap"
 
  "github.com/elastic/package-registry/packages"
+ "github.com/elastic/package-registry/proxymode"
  "github.com/elastic/package-registry/util"
 )
 
@@ -22,6 +23,10 @@ const artifactsRouterPath = "/epr/{packageName}/{packageName:[a-z0-9_]+}-{packag
 var errArtifactNotFound = errors.New("artifact not found")
 
 func artifactsHandler(indexer Indexer, cacheTime time.Duration) func(w http.ResponseWriter, r *http.Request) {
+ return artifactsHandlerWithProxyMode(indexer, proxymode.NoProxy(), cacheTime)
+}
+
+func artifactsHandlerWithProxyMode(indexer Indexer, proxyMode *proxymode.ProxyMode, cacheTime time.Duration) func(w http.ResponseWriter, r *http.Request) {
  logger := util.Logger()
  return func(w http.ResponseWriter, r *http.Request) {
  vars := mux.Vars(r)
@@ -44,7 +49,7 @@ func artifactsHandler(indexer Indexer, cacheTime time.Duration) func(w http.Resp
  }
 
  opts := packages.NameVersionFilter(packageName, packageVersion)
- packageList, err := indexer.Get(r.Context(), &opts)
+ pkgs, err := indexer.Get(r.Context(), &opts)
  if err != nil {
  logger.Error("getting package path failed",
  zap.String("package.name", packageName),
@@ -53,12 +58,21 @@ func artifactsHandler(indexer Indexer, cacheTime time.Duration) func(w http.Resp
  http.Error(w, "internal server error", http.StatusInternalServerError)
  return
  }
- if len(packageList) == 0 {
+ if len(pkgs) == 0 && proxyMode.Enabled() {
+ proxiedPackage, err := proxyMode.Package(r)
+ if err != nil {
+ logger.Error("proxy mode: package failed", zap.Error(err))
+ http.Error(w, "internal server error", http.StatusInternalServerError)
+ return
+ }
+ pkgs = pkgs.Join(packages.Packages{proxiedPackage})
+ }
+ if len(pkgs) == 0 {
  notFoundError(w, errArtifactNotFound)
  return
  }
 
  cacheHeaders(w, cacheTime)
- packages.ServePackage(w, r, packageList[0])
+ packages.ServePackage(w, r, pkgs[0])
  }
 }

categories.go

@@ -13,21 +13,24 @@ import (
  "strconv"
  "time"
 
+ "go.uber.org/zap"
+
  "github.com/Masterminds/semver/v3"
  "go.elastic.co/apm"
 
  "github.com/elastic/package-registry/packages"
+ "github.com/elastic/package-registry/proxymode"
  "github.com/elastic/package-registry/util"
 )
 
-type Category struct {
- Id string `yaml:"id" json:"id"`
- Title string `yaml:"title" json:"title"`
- Count int `yaml:"count" json:"count"`
+// categoriesHandler is a dynamic handler as it will also allow filtering in the future.
+func categoriesHandler(indexer Indexer, cacheTime time.Duration) func(w http.ResponseWriter, r *http.Request) {
+ return categoriesHandlerWithProxyMode(indexer, proxymode.NoProxy(), cacheTime)
 }
 
 // categoriesHandler is a dynamic handler as it will also allow filtering in the future.
-func categoriesHandler(indexer Indexer, cacheTime time.Duration) func(w http.ResponseWriter, r *http.Request) {
+func categoriesHandlerWithProxyMode(indexer Indexer, proxyMode *proxymode.ProxyMode, cacheTime time.Duration) func(w http.ResponseWriter, r *http.Request) {
+ logger := util.Logger()
  return func(w http.ResponseWriter, r *http.Request) {
  query := r.URL.Query()
 
@@ -49,13 +52,33 @@ func categoriesHandler(indexer Indexer, cacheTime time.Duration) func(w http.Res
  opts := packages.GetOptions{
  Filter: filter,
  }
- packages, err := indexer.Get(r.Context(), &opts)
+ pkgs, err := indexer.Get(r.Context(), &opts)
  if err != nil {
  notFoundError(w, err)
  return
  }
+ categories := getCategories(r.Context(), pkgs, includePolicyTemplates)
+
+ if proxyMode.Enabled() {
+ proxiedCategories, err := proxyMode.Categories(r)
+ if err != nil {
+ logger.Error("proxy mode: categories failed", zap.Error(err))
+ http.Error(w, "internal server error", http.StatusInternalServerError)
+ return
+ }
 
- categories := getCategories(r.Context(), packages, includePolicyTemplates)
+ for _, category := range proxiedCategories {
+ if _, ok := categories[category.Id]; !ok {
+ categories[category.Id] = &packages.Category{
+ Id: category.Id,
+ Title: category.Title,
+ Count: category.Count,
+ }
+ } else {
+ categories[category.Id].Count += category.Count
+ }
+ }
+ }
 
  data, err := getCategoriesOutput(r.Context(), categories)
  if err != nil {
@@ -108,16 +131,16 @@ func newCategoriesFilterFromQuery(query url.Values) (*packages.Filter, error) {
  return &filter, nil
 }
 
-func getCategories(ctx context.Context, packages packages.Packages, includePolicyTemplates bool) map[string]*Category {
+func getCategories(ctx context.Context, pkgs packages.Packages, includePolicyTemplates bool) map[string]*packages.Category {
  span, ctx := apm.StartSpan(ctx, "FilterCategories", "app")
  defer span.End()
 
- categories := map[string]*Category{}
+ categories := map[string]*packages.Category{}
 
- for _, p := range packages {
+ for _, p := range pkgs {
  for _, c := range p.Categories {
  if _, ok := categories[c]; !ok {
- categories[c] = &Category{
+ categories[c] = &packages.Category{
  Id: c,
  Title: c,
  Count: 0,
@@ -146,7 +169,7 @@ func getCategories(ctx context.Context, packages packages.Packages, includePolic
  // Add policy template level categories.
  for _, c := range t.Categories {
  if _, ok := categories[c]; !ok {
- categories[c] = &Category{
+ categories[c] = &packages.Category{
  Id: c,
  Title: c,
  Count: 0,
@@ -167,7 +190,7 @@ func getCategories(ctx context.Context, packages packages.Packages, includePolic
  return categories
 }
 
-func getCategoriesOutput(ctx context.Context, categories map[string]*Category) ([]byte, error) {
+func getCategoriesOutput(ctx context.Context, categories map[string]*packages.Category) ([]byte, error) {
  span, ctx := apm.StartSpan(ctx, "GetCategoriesOutput", "app")
  defer span.End()
 
@@ -177,7 +200,7 @@ func getCategoriesOutput(ctx context.Context, categories map[string]*Category) (
  }
  sort.Strings(keys)
 
- var outputCategories []*Category
+ var outputCategories []*packages.Category
  for _, k := range keys {
  c := categories[k]
  if title, ok := packages.CategoryTitles[c.Title]; ok {

main.go

@@ -8,6 +8,7 @@ import (
  "context"
  "flag"
  "fmt"
+ "log"
  "net/http"
  _ "net/http/pprof"
  "os"
@@ -29,6 +30,7 @@ import (
 
  "github.com/elastic/package-registry/metrics"
  "github.com/elastic/package-registry/packages"
+ "github.com/elastic/package-registry/proxymode"
  "github.com/elastic/package-registry/storage"
  "github.com/elastic/package-registry/util"
 )
@@ -57,6 +59,9 @@ var (
  storageEndpoint string
  storageIndexerWatchInterval time.Duration
 
+ featureProxyMode bool
+ proxyTo string
+
  defaultConfig = Config{
  CacheTimeIndex: 10 * time.Second,
  CacheTimeSearch: 10 * time.Minute,
@@ -82,6 +87,9 @@ func init() {
  flag.StringVar(&storageEndpoint, "storage-endpoint", "https://package-storage.elastic.co/", "Package Storage public endpoint.")
  flag.DurationVar(&storageIndexerWatchInterval, "storage-indexer-watch-interval", 1*time.Minute, "Address of the package-registry service.")
 
+ // The following proxy-indexer related flags are technical preview and might be removed in the future or renamed
+ flag.BoolVar(&featureProxyMode, "feature-proxy-mode", false, "Enable proxy mode to include packages from other endpoint (technical preview).")
+ flag.StringVar(&proxyTo, "proxy-to", "https://epr-snapshot.elastic.co/", "Proxy-to endpoint")
 }
 
 type Config struct {
@@ -180,26 +188,26 @@ func initMetricsServer(logger *zap.Logger) {
 func initIndexer(ctx context.Context, logger *zap.Logger, config *Config) Indexer {
  packagesBasePaths := getPackagesBasePaths(config)
 
- var indexer Indexer
+ var combined CombinedIndexer
+
  if featureStorageIndexer {
  storageClient, err := gstorage.NewClient(ctx)
  if err != nil {
  logger.Fatal("can't initialize storage client", zap.Error(err))
  }
- indexer = storage.NewIndexer(storageClient, storage.IndexerOptions{
+ combined = append(combined, storage.NewIndexer(storageClient, storage.IndexerOptions{
  PackageStorageBucketInternal: storageIndexerBucketInternal,
  PackageStorageEndpoint: storageEndpoint,
  WatchInterval: storageIndexerWatchInterval,
- })
- } else {
- indexer = NewCombinedIndexer(
- packages.NewZipFileSystemIndexer(packagesBasePaths...),
- packages.NewFileSystemIndexer(packagesBasePaths...),
- )
+ }))
  }
- ensurePackagesAvailable(ctx, logger, indexer)
 
- return indexer
+ combined = append(combined,
+ packages.NewZipFileSystemIndexer(packagesBasePaths...),
+ packages.NewFileSystemIndexer(packagesBasePaths...),
+ )
+ ensurePackagesAvailable(ctx, logger, combined)
+ return combined
 }
 
 func initServer(logger *zap.Logger, config *Config) *http.Server {
@@ -310,8 +318,18 @@ func mustLoadRouter(logger *zap.Logger, config *Config, indexer Indexer) *mux.Ro
 }
 
 func getRouter(logger *zap.Logger, config *Config, indexer Indexer) (*mux.Router, error) {
- artifactsHandler := artifactsHandler(indexer, config.CacheTimeCatchAll)
- signaturesHandler := signaturesHandler(indexer, config.CacheTimeCatchAll)
+ if featureProxyMode {
+ log.Println("Technical preview: Proxy mode is an experimental feature and it may be unstable.")
+ }
+ proxyMode, err := proxymode.NewProxyMode(proxymode.ProxyOptions{
+ Enabled: featureProxyMode,
+ ProxyTo: proxyTo,
+ })
+ if err != nil {
+ return nil, errors.Wrapf(err, "can't create proxy mode")
+ }
+ artifactsHandler := artifactsHandlerWithProxyMode(indexer, proxyMode, config.CacheTimeCatchAll)
+ signaturesHandler := signaturesHandlerWithProxyMode(indexer, proxyMode, config.CacheTimeCatchAll)
  faviconHandleFunc, err := faviconHandler(config.CacheTimeCatchAll)
  if err != nil {
  return nil, err
@@ -321,14 +339,16 @@ func getRouter(logger *zap.Logger, config *Config, indexer Indexer) (*mux.Router
  return nil, err
  }
 
- packageIndexHandler := packageIndexHandler(indexer, config.CacheTimeCatchAll)
- staticHandler := staticHandler(indexer, config.CacheTimeCatchAll)
+ categoriesHandler := categoriesHandlerWithProxyMode(indexer, proxyMode, config.CacheTimeCategories)
+ packageIndexHandler := packageIndexHandlerWithProxyMode(indexer, proxyMode, config.CacheTimeCatchAll)
+ searchHandler := searchHandlerWithProxyMode(indexer, proxyMode, config.CacheTimeSearch)
+ staticHandler := staticHandlerWithProxyMode(indexer, proxyMode, config.CacheTimeCatchAll)
 
  router := mux.NewRouter().StrictSlash(true)
  router.HandleFunc("/", indexHandlerFunc)
  router.HandleFunc("/index.json", indexHandlerFunc)
- router.HandleFunc("/search", searchHandler(indexer, config.CacheTimeSearch))
- router.HandleFunc("/categories", categoriesHandler(indexer, config.CacheTimeCategories))
+ router.HandleFunc("/search", searchHandler)
+ router.HandleFunc("/categories", categoriesHandler)
  router.HandleFunc("/health", healthHandler)
  router.HandleFunc("/favicon.ico", faviconHandleFunc)
  router.HandleFunc(artifactsRouterPath, artifactsHandler)
@@ -339,7 +359,7 @@ func getRouter(logger *zap.Logger, config *Config, indexer Indexer) (*mux.Router
  if metricsAddress != "" {
  router.Use(metrics.MetricsMiddleware())
  }
- router.NotFoundHandler = http.Handler(notFoundHandler(fmt.Errorf("404 page not found")))
+ router.NotFoundHandler = notFoundHandler(fmt.Errorf("404 page not found"))
  return router, nil
 }
 

package_index.go

@@ -15,6 +15,7 @@ import (
  "github.com/gorilla/mux"
 
  "github.com/elastic/package-registry/packages"
+ "github.com/elastic/package-registry/proxymode"
  "github.com/elastic/package-registry/util"
 )
 
@@ -25,6 +26,10 @@ const (
 var errPackageRevisionNotFound = errors.New("package revision not found")
 
 func packageIndexHandler(indexer Indexer, cacheTime time.Duration) func(w http.ResponseWriter, r *http.Request) {
+ return packageIndexHandlerWithProxyMode(indexer, proxymode.NoProxy(), cacheTime)
+}
+
+func packageIndexHandlerWithProxyMode(indexer Indexer, proxyMode *proxymode.ProxyMode, cacheTime time.Duration) func(w http.ResponseWriter, r *http.Request) {
  logger := util.Logger()
  return func(w http.ResponseWriter, r *http.Request) {
  vars := mux.Vars(r)
@@ -47,24 +52,33 @@ func packageIndexHandler(indexer Indexer, cacheTime time.Duration) func(w http.R
  }
 
  opts := packages.NameVersionFilter(packageName, packageVersion)
- packages, err := indexer.Get(r.Context(), &opts)
+ pkgs, err := indexer.Get(r.Context(), &opts)
  if err != nil {
  logger.Error("getting package path failed", zap.Error(err))
  http.Error(w, "internal server error", http.StatusInternalServerError)
  return
  }
- if len(packages) == 0 {
+ if len(pkgs) == 0 && proxyMode.Enabled() {
+ proxiedPackage, err := proxyMode.Package(r)
+ if err != nil {
+ logger.Error("proxy mode: package failed", zap.Error(err))
+ http.Error(w, "internal server error", http.StatusInternalServerError)
+ return
+ }
+ pkgs = pkgs.Join(packages.Packages{proxiedPackage})
+ }
+ if len(pkgs) == 0 {
  notFoundError(w, errPackageRevisionNotFound)
  return
  }
 
  w.Header().Set("Content-Type", "application/json")
  cacheHeaders(w, cacheTime)
 
- err = util.WriteJSONPretty(w, packages[0])
+ err = util.WriteJSONPretty(w, pkgs[0])
  if err != nil {
  logger.Error("marshaling package index failed",
- zap.String("package.path", packages[0].BasePath),
+ zap.String("package.path", pkgs[0].BasePath),
  zap.Error(err))
  http.Error(w, "internal server error", http.StatusInternalServerError)
  return

packages/category.go

@@ -0,0 +1,11 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package packages
+
+type Category struct {
+ Id string `yaml:"id" json:"id"`
+ Title string `yaml:"title" json:"title"`
+ Count int `yaml:"count" json:"count"`
+}