Implement proxy mode (#860)

* WIP

* Refactor indexer to proxy mode

* Prepare interfaces

* Link

* Bump github.com/prometheus/client_golang from 1.12.2 to 1.13.0 (#861)

Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.12.2 to 1.13.0.
- [Release notes](https://github.com/prometheus/client_golang/releases)
- [Changelog](https://github.com/prometheus/client_golang/blob/main/CHANGELOG.md)
- [Commits](prometheus/client_golang@v1.12.2...v1.13.0)

---
updated-dependencies:
- dependency-name: github.com/prometheus/client_golang
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* Bump go.uber.org/zap from 1.21.0 to 1.22.0 (#862)

Bumps [go.uber.org/zap](https://github.com/uber-go/zap) from 1.21.0 to 1.22.0.
- [Release notes](https://github.com/uber-go/zap/releases)
- [Changelog](https://github.com/uber-go/zap/blob/master/CHANGELOG.md)
- [Commits](uber-go/zap@v1.21.0...v1.22.0)

---
updated-dependencies:
- dependency-name: go.uber.org/zap
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* Update favicon with the new EPR logo (#858)

Update favicon file with the new Elastic Package Registry logo.
A new "img" folder has also been added to include other
available formats of this new logo (png, svg and ico).

* Bump cloud.google.com/go/storage from 1.24.0 to 1.25.0 (#863)

Bumps [cloud.google.com/go/storage](https://github.com/googleapis/google-cloud-go) from 1.24.0 to 1.25.0.
- [Release notes](https://github.com/googleapis/google-cloud-go/releases)
- [Changelog](https://github.com/googleapis/google-cloud-go/blob/main/CHANGES.md)
- [Commits](googleapis/google-cloud-go@pubsub/v1.24.0...spanner/v1.25.0)

---
updated-dependencies:
- dependency-name: cloud.google.com/go/storage
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* ci: transform git tag with `v` format (#865)

* Bump github.com/fsouza/fake-gcs-server from 1.38.3 to 1.38.4 (#866)

Bumps [github.com/fsouza/fake-gcs-server](https://github.com/fsouza/fake-gcs-server) from 1.38.3 to 1.38.4.
- [Release notes](https://github.com/fsouza/fake-gcs-server/releases)
- [Commits](fsouza/fake-gcs-server@v1.38.3...v1.38.4)

---
updated-dependencies:
- dependency-name: github.com/fsouza/fake-gcs-server
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* Use proxymode in main

* Search

* WIP: before resolver

* Resolver

* Basic resolver

* Fix: mage check

* Not needed

* Fix: imports

* Add CHANGELOG

* Categories

* Address PR comments

* Use consts

* More fixes

* Use snapshot registry

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Co-authored-by: Victor Martinez <victormartinezrubio@gmail.com>

CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 * Update favicon to be the Elastic Package Registry logo. [#858](https://github.com/elastic/package-registry/pull/858)
+* Implement proxy mode. [#860](https://github.com/elastic/package-registry/pull/860)
 
 ### Deprecated
 

artifacts.go

@@ -14,6 +14,7 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/elastic/package-registry/packages"
+	"github.com/elastic/package-registry/proxymode"
 	"github.com/elastic/package-registry/util"
 )
 
@@ -22,6 +23,10 @@ const artifactsRouterPath = "/epr/{packageName}/{packageName:[a-z0-9_]+}-{packag
 var errArtifactNotFound = errors.New("artifact not found")
 
 func artifactsHandler(indexer Indexer, cacheTime time.Duration) func(w http.ResponseWriter, r *http.Request) {
+	return artifactsHandlerWithProxyMode(indexer, proxymode.NoProxy(), cacheTime)
+}
+
+func artifactsHandlerWithProxyMode(indexer Indexer, proxyMode *proxymode.ProxyMode, cacheTime time.Duration) func(w http.ResponseWriter, r *http.Request) {
 	logger := util.Logger()
 	return func(w http.ResponseWriter, r *http.Request) {
 		vars := mux.Vars(r)
@@ -44,7 +49,7 @@ func artifactsHandler(indexer Indexer, cacheTime time.Duration) func(w http.Resp
 		}
 
 		opts := packages.NameVersionFilter(packageName, packageVersion)
-		packageList, err := indexer.Get(r.Context(), &opts)
+		pkgs, err := indexer.Get(r.Context(), &opts)
 		if err != nil {
 			logger.Error("getting package path failed",
 				zap.String("package.name", packageName),
@@ -53,12 +58,21 @@ func artifactsHandler(indexer Indexer, cacheTime time.Duration) func(w http.Resp
 			http.Error(w, "internal server error", http.StatusInternalServerError)
 			return
 		}
-		if len(packageList) == 0 {
+		if len(pkgs) == 0 && proxyMode.Enabled() {
+			proxiedPackage, err := proxyMode.Package(r)
+			if err != nil {
+				logger.Error("proxy mode: package failed", zap.Error(err))
+				http.Error(w, "internal server error", http.StatusInternalServerError)
+				return
+			}
+			pkgs = pkgs.Join(packages.Packages{proxiedPackage})
+		}
+		if len(pkgs) == 0 {
 			notFoundError(w, errArtifactNotFound)
 			return
 		}
 
 		cacheHeaders(w, cacheTime)
-		packages.ServePackage(w, r, packageList[0])
+		packages.ServePackage(w, r, pkgs[0])
 	}
 }

categories.go

@@ -13,21 +13,24 @@ import (
 	"strconv"
 	"time"
 
+	"go.uber.org/zap"
+
 	"github.com/Masterminds/semver/v3"
 	"go.elastic.co/apm"
 
 	"github.com/elastic/package-registry/packages"
+	"github.com/elastic/package-registry/proxymode"
 	"github.com/elastic/package-registry/util"
 )
 
-type Category struct {
-	Id    string `yaml:"id" json:"id"`
-	Title string `yaml:"title" json:"title"`
-	Count int    `yaml:"count" json:"count"`
+// categoriesHandler is a dynamic handler as it will also allow filtering in the future.
+func categoriesHandler(indexer Indexer, cacheTime time.Duration) func(w http.ResponseWriter, r *http.Request) {
+	return categoriesHandlerWithProxyMode(indexer, proxymode.NoProxy(), cacheTime)
 }
 
 // categoriesHandler is a dynamic handler as it will also allow filtering in the future.
-func categoriesHandler(indexer Indexer, cacheTime time.Duration) func(w http.ResponseWriter, r *http.Request) {
+func categoriesHandlerWithProxyMode(indexer Indexer, proxyMode *proxymode.ProxyMode, cacheTime time.Duration) func(w http.ResponseWriter, r *http.Request) {
+	logger := util.Logger()
 	return func(w http.ResponseWriter, r *http.Request) {
 		query := r.URL.Query()
 
@@ -49,13 +52,33 @@ func categoriesHandler(indexer Indexer, cacheTime time.Duration) func(w http.Res
 		opts := packages.GetOptions{
 			Filter: filter,
 		}
-		packages, err := indexer.Get(r.Context(), &opts)
+		pkgs, err := indexer.Get(r.Context(), &opts)
 		if err != nil {
 			notFoundError(w, err)
 			return
 		}
+		categories := getCategories(r.Context(), pkgs, includePolicyTemplates)
+
+		if proxyMode.Enabled() {
+			proxiedCategories, err := proxyMode.Categories(r)
+			if err != nil {
+				logger.Error("proxy mode: categories failed", zap.Error(err))
+				http.Error(w, "internal server error", http.StatusInternalServerError)
+				return
+			}
 
-		categories := getCategories(r.Context(), packages, includePolicyTemplates)
+			for _, category := range proxiedCategories {
+				if _, ok := categories[category.Id]; !ok {
+					categories[category.Id] = &packages.Category{
+						Id:    category.Id,
+						Title: category.Title,
+						Count: category.Count,
+					}
+				} else {
+					categories[category.Id].Count += category.Count
+				}
+			}
+		}
 
 		data, err := getCategoriesOutput(r.Context(), categories)
 		if err != nil {
@@ -108,16 +131,16 @@ func newCategoriesFilterFromQuery(query url.Values) (*packages.Filter, error) {
 	return &filter, nil
 }
 
-func getCategories(ctx context.Context, packages packages.Packages, includePolicyTemplates bool) map[string]*Category {
+func getCategories(ctx context.Context, pkgs packages.Packages, includePolicyTemplates bool) map[string]*packages.Category {
 	span, ctx := apm.StartSpan(ctx, "FilterCategories", "app")
 	defer span.End()
 
-	categories := map[string]*Category{}
+	categories := map[string]*packages.Category{}
 
-	for _, p := range packages {
+	for _, p := range pkgs {
 		for _, c := range p.Categories {
 			if _, ok := categories[c]; !ok {
-				categories[c] = &Category{
+				categories[c] = &packages.Category{
 					Id:    c,
 					Title: c,
 					Count: 0,
@@ -146,7 +169,7 @@ func getCategories(ctx context.Context, packages packages.Packages, includePolic
 				// Add policy template level categories.
 				for _, c := range t.Categories {
 					if _, ok := categories[c]; !ok {
-						categories[c] = &Category{
+						categories[c] = &packages.Category{
 							Id:    c,
 							Title: c,
 							Count: 0,
@@ -167,7 +190,7 @@ func getCategories(ctx context.Context, packages packages.Packages, includePolic
 	return categories
 }
 
-func getCategoriesOutput(ctx context.Context, categories map[string]*Category) ([]byte, error) {
+func getCategoriesOutput(ctx context.Context, categories map[string]*packages.Category) ([]byte, error) {
 	span, ctx := apm.StartSpan(ctx, "GetCategoriesOutput", "app")
 	defer span.End()
 
@@ -177,7 +200,7 @@ func getCategoriesOutput(ctx context.Context, categories map[string]*Category) (
 	}
 	sort.Strings(keys)
 
-	var outputCategories []*Category
+	var outputCategories []*packages.Category
 	for _, k := range keys {
 		c := categories[k]
 		if title, ok := packages.CategoryTitles[c.Title]; ok {

main.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"log"
 	"net/http"
 	_ "net/http/pprof"
 	"os"
@@ -29,6 +30,7 @@ import (
 
 	"github.com/elastic/package-registry/metrics"
 	"github.com/elastic/package-registry/packages"
+	"github.com/elastic/package-registry/proxymode"
 	"github.com/elastic/package-registry/storage"
 	"github.com/elastic/package-registry/util"
 )
@@ -57,6 +59,9 @@ var (
 	storageEndpoint              string
 	storageIndexerWatchInterval  time.Duration
 
+	featureProxyMode bool
+	proxyTo          string
+
 	defaultConfig = Config{
 		CacheTimeIndex:      10 * time.Second,
 		CacheTimeSearch:     10 * time.Minute,
@@ -82,6 +87,9 @@ func init() {
 	flag.StringVar(&storageEndpoint, "storage-endpoint", "https://package-storage.elastic.co/", "Package Storage public endpoint.")
 	flag.DurationVar(&storageIndexerWatchInterval, "storage-indexer-watch-interval", 1*time.Minute, "Address of the package-registry service.")
 
+	// The following proxy-indexer related flags are technical preview and might be removed in the future or renamed
+	flag.BoolVar(&featureProxyMode, "feature-proxy-mode", false, "Enable proxy mode to include packages from other endpoint (technical preview).")
+	flag.StringVar(&proxyTo, "proxy-to", "https://epr-snapshot.elastic.co/", "Proxy-to endpoint")
 }
 
 type Config struct {
@@ -180,26 +188,26 @@ func initMetricsServer(logger *zap.Logger) {
 func initIndexer(ctx context.Context, logger *zap.Logger, config *Config) Indexer {
 	packagesBasePaths := getPackagesBasePaths(config)
 
-	var indexer Indexer
+	var combined CombinedIndexer
+
 	if featureStorageIndexer {
 		storageClient, err := gstorage.NewClient(ctx)
 		if err != nil {
 			logger.Fatal("can't initialize storage client", zap.Error(err))
 		}
-		indexer = storage.NewIndexer(storageClient, storage.IndexerOptions{
+		combined = append(combined, storage.NewIndexer(storageClient, storage.IndexerOptions{
 			PackageStorageBucketInternal: storageIndexerBucketInternal,
 			PackageStorageEndpoint:       storageEndpoint,
 			WatchInterval:                storageIndexerWatchInterval,
-		})
-	} else {
-		indexer = NewCombinedIndexer(
-			packages.NewZipFileSystemIndexer(packagesBasePaths...),
-			packages.NewFileSystemIndexer(packagesBasePaths...),
-		)
+		}))
 	}
-	ensurePackagesAvailable(ctx, logger, indexer)
 
-	return indexer
+	combined = append(combined,
+		packages.NewZipFileSystemIndexer(packagesBasePaths...),
+		packages.NewFileSystemIndexer(packagesBasePaths...),
+	)
+	ensurePackagesAvailable(ctx, logger, combined)
+	return combined
 }
 
 func initServer(logger *zap.Logger, config *Config) *http.Server {
@@ -310,8 +318,18 @@ func mustLoadRouter(logger *zap.Logger, config *Config, indexer Indexer) *mux.Ro
 }
 
 func getRouter(logger *zap.Logger, config *Config, indexer Indexer) (*mux.Router, error) {
-	artifactsHandler := artifactsHandler(indexer, config.CacheTimeCatchAll)
-	signaturesHandler := signaturesHandler(indexer, config.CacheTimeCatchAll)
+	if featureProxyMode {
+		log.Println("Technical preview: Proxy mode is an experimental feature and it may be unstable.")
+	}
+	proxyMode, err := proxymode.NewProxyMode(proxymode.ProxyOptions{
+		Enabled: featureProxyMode,
+		ProxyTo: proxyTo,
+	})
+	if err != nil {
+		return nil, errors.Wrapf(err, "can't create proxy mode")
+	}
+	artifactsHandler := artifactsHandlerWithProxyMode(indexer, proxyMode, config.CacheTimeCatchAll)
+	signaturesHandler := signaturesHandlerWithProxyMode(indexer, proxyMode, config.CacheTimeCatchAll)
 	faviconHandleFunc, err := faviconHandler(config.CacheTimeCatchAll)
 	if err != nil {
 		return nil, err
@@ -321,14 +339,16 @@ func getRouter(logger *zap.Logger, config *Config, indexer Indexer) (*mux.Router
 		return nil, err
 	}
 
-	packageIndexHandler := packageIndexHandler(indexer, config.CacheTimeCatchAll)
-	staticHandler := staticHandler(indexer, config.CacheTimeCatchAll)
+	categoriesHandler := categoriesHandlerWithProxyMode(indexer, proxyMode, config.CacheTimeCategories)
+	packageIndexHandler := packageIndexHandlerWithProxyMode(indexer, proxyMode, config.CacheTimeCatchAll)
+	searchHandler := searchHandlerWithProxyMode(indexer, proxyMode, config.CacheTimeSearch)
+	staticHandler := staticHandlerWithProxyMode(indexer, proxyMode, config.CacheTimeCatchAll)
 
 	router := mux.NewRouter().StrictSlash(true)
 	router.HandleFunc("/", indexHandlerFunc)
 	router.HandleFunc("/index.json", indexHandlerFunc)
-	router.HandleFunc("/search", searchHandler(indexer, config.CacheTimeSearch))
-	router.HandleFunc("/categories", categoriesHandler(indexer, config.CacheTimeCategories))
+	router.HandleFunc("/search", searchHandler)
+	router.HandleFunc("/categories", categoriesHandler)
 	router.HandleFunc("/health", healthHandler)
 	router.HandleFunc("/favicon.ico", faviconHandleFunc)
 	router.HandleFunc(artifactsRouterPath, artifactsHandler)
@@ -339,7 +359,7 @@ func getRouter(logger *zap.Logger, config *Config, indexer Indexer) (*mux.Router
 	if metricsAddress != "" {
 		router.Use(metrics.MetricsMiddleware())
 	}
-	router.NotFoundHandler = http.Handler(notFoundHandler(fmt.Errorf("404 page not found")))
+	router.NotFoundHandler = notFoundHandler(fmt.Errorf("404 page not found"))
 	return router, nil
 }
 

package_index.go

@@ -15,6 +15,7 @@ import (
 	"github.com/gorilla/mux"
 
 	"github.com/elastic/package-registry/packages"
+	"github.com/elastic/package-registry/proxymode"
 	"github.com/elastic/package-registry/util"
 )
 
@@ -25,6 +26,10 @@ const (
 var errPackageRevisionNotFound = errors.New("package revision not found")
 
 func packageIndexHandler(indexer Indexer, cacheTime time.Duration) func(w http.ResponseWriter, r *http.Request) {
+	return packageIndexHandlerWithProxyMode(indexer, proxymode.NoProxy(), cacheTime)
+}
+
+func packageIndexHandlerWithProxyMode(indexer Indexer, proxyMode *proxymode.ProxyMode, cacheTime time.Duration) func(w http.ResponseWriter, r *http.Request) {
 	logger := util.Logger()
 	return func(w http.ResponseWriter, r *http.Request) {
 		vars := mux.Vars(r)
@@ -47,24 +52,33 @@ func packageIndexHandler(indexer Indexer, cacheTime time.Duration) func(w http.R
 		}
 
 		opts := packages.NameVersionFilter(packageName, packageVersion)
-		packages, err := indexer.Get(r.Context(), &opts)
+		pkgs, err := indexer.Get(r.Context(), &opts)
 		if err != nil {
 			logger.Error("getting package path failed", zap.Error(err))
 			http.Error(w, "internal server error", http.StatusInternalServerError)
 			return
 		}
-		if len(packages) == 0 {
+		if len(pkgs) == 0 && proxyMode.Enabled() {
+			proxiedPackage, err := proxyMode.Package(r)
+			if err != nil {
+				logger.Error("proxy mode: package failed", zap.Error(err))
+				http.Error(w, "internal server error", http.StatusInternalServerError)
+				return
+			}
+			pkgs = pkgs.Join(packages.Packages{proxiedPackage})
+		}
+		if len(pkgs) == 0 {
 			notFoundError(w, errPackageRevisionNotFound)
 			return
 		}
 
 		w.Header().Set("Content-Type", "application/json")
 		cacheHeaders(w, cacheTime)
 
-		err = util.WriteJSONPretty(w, packages[0])
+		err = util.WriteJSONPretty(w, pkgs[0])
 		if err != nil {
 			logger.Error("marshaling package index failed",
-				zap.String("package.path", packages[0].BasePath),
+				zap.String("package.path", pkgs[0].BasePath),
 				zap.Error(err))
 			http.Error(w, "internal server error", http.StatusInternalServerError)
 			return

packages/category.go

@@ -0,0 +1,11 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package packages
+
+type Category struct {
+	Id    string `yaml:"id" json:"id"`
+	Title string `yaml:"title" json:"title"`
+	Count int    `yaml:"count" json:"count"`
+}