Merge pull request #626 from souleb/reuse-index-pool

Add optional in-memory cache of HelmRepository index files

api/v1beta2/condition_types.go

@@ -97,4 +97,7 @@ const (
  // ArtifactUpToDateReason signals that an existing Artifact is up-to-date
  // with the Source.
  ArtifactUpToDateReason string = "ArtifactUpToDate"
+
+ // CacheOperationFailedReason signals a failure in cache operation.
+ CacheOperationFailedReason string = "CacheOperationFailed"
 )

controllers/helmchart_controller.go

@@ -30,6 +30,7 @@ import (
 
  securejoin "github.com/cyphar/filepath-securejoin"
  helmgetter "helm.sh/helm/v3/pkg/getter"
+ helmrepo "helm.sh/helm/v3/pkg/repo"
  corev1 "k8s.io/api/core/v1"
  apierrs "k8s.io/apimachinery/pkg/api/errors"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -55,6 +56,7 @@ import (
  "github.com/fluxcd/pkg/untar"
 
  sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
+ "github.com/fluxcd/source-controller/internal/cache"
  serror "github.com/fluxcd/source-controller/internal/error"
  "github.com/fluxcd/source-controller/internal/helm/chart"
  "github.com/fluxcd/source-controller/internal/helm/getter"
@@ -111,6 +113,9 @@ type HelmChartReconciler struct {
  Storage *Storage
  Getters helmgetter.Providers
  ControllerName string
+
+ Cache *cache.Cache
+ TTL time.Duration
 }
 
 func (r *HelmChartReconciler) SetupWithManager(mgr ctrl.Manager) error {
@@ -456,6 +461,13 @@ func (r *HelmChartReconciler) buildFromHelmRepository(ctx context.Context, obj *
  }
  }
 
+ // Try to retrieve the repository index from the cache
+ if r.Cache != nil {
+ if index, found := r.Cache.Get(r.Storage.LocalPath(*repo.GetArtifact())); found {
+ chartRepo.Index = index.(*helmrepo.IndexFile)
+ }
+ }
+
  // Construct the chart builder with scoped configuration
  cb := chart.NewRemoteBuilder(chartRepo)
  opts := chart.BuildOptions{
@@ -479,6 +491,26 @@ func (r *HelmChartReconciler) buildFromHelmRepository(ctx context.Context, obj *
  return sreconcile.ResultEmpty, err
  }
 
+ defer func() {
+ // Cache the index if it was successfully retrieved
+ // and the chart was successfully built
+ if r.Cache != nil && chartRepo.Index != nil {
+ // The cache key have to be safe in multi-tenancy environments,
+ // as otherwise it could be used as a vector to bypass the helm repository's authentication.
+ // Using r.Storage.LocalPath(*repo.GetArtifact() is safe as the path is in the format /<helm-repository-name>/<chart-name>/<filename>.
+ err := r.Cache.Set(r.Storage.LocalPath(*repo.GetArtifact()), chartRepo.Index, r.TTL)
+ if err != nil {
+ r.eventLogf(ctx, obj, events.EventTypeTrace, sourcev1.CacheOperationFailedReason, "failed to cache index: %s", err)
+ }
+
+ }
+
+ // Delete the index reference
+ if chartRepo.Index != nil {
+ chartRepo.Unload()
+ }
+ }()
+
  *b = *build
  return sreconcile.ResultSuccess, nil
 }

controllers/suite_test.go

@@ -35,6 +35,7 @@ import (
  "github.com/fluxcd/pkg/testserver"
 
  sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
+ "github.com/fluxcd/source-controller/internal/cache"
  // +kubebuilder:scaffold:imports
 )
 
@@ -126,12 +127,15 @@ func TestMain(m *testing.M) {
  panic(fmt.Sprintf("Failed to start HelmRepositoryReconciler: %v", err))
  }
 
+ cache := cache.New(5, 1*time.Second)
  if err := (&HelmChartReconciler{
  Client: testEnv,
  EventRecorder: record.NewFakeRecorder(32),
  Metrics: testMetricsH,
  Getters: testGetters,
  Storage: testStorage,
+ Cache: cache,
+ TTL: 1 * time.Second,
  }).SetupWithManager(testEnv); err != nil {
  panic(fmt.Sprintf("Failed to start HelmRepositoryReconciler: %v", err))
  }

docs/spec/v1beta2/helmcharts.md

@@ -390,6 +390,53 @@ Besides being reported in Events, the reconciliation errors are also logged by
 the controller. The Flux CLI offer commands for filtering the logs for a
 specific HelmChart, e.g. `flux logs --level=error --kind=HelmChart --name=<chart-name>`.
 
+### Improving resource consumption by enabling the cache
+
+When using a `HelmRepository` as Source for a `HelmChart`, the controller loads
+the repository index in memory to find the latest version of the chart.
+
+The controller can be configured to cache Helm repository indexes in memory.
+The cache is used to avoid loading repository indexes for every `HelmChart`
+reconciliation.
+
+The following flags are provided to enable and configure the cache:
+- `helm-cache-max-size`: The maximum size of the cache in number of indexes.
+ If `0`, then the cache is disabled.
+- `helm-cache-ttl`: The TTL of an index in the cache.
+- `helm-cache-purge-interval`: The interval at which the cache is purged of
+ expired items. 
+
+The caching strategy is to pull a repository index from the cache if it is 
+available, otherwise to load the index, retrieve and build the chart,
+then cache the index. The cached index TTL is refreshed every time the
+Helm repository index is loaded with the `helm-cache-ttl` value.
+
+The cache is purged of expired items every `helm-cache-purge-interval`.
+
+When the cache is full, no more items can be added to the cache, and the
+source-controller will report a warning event instead.
+
+In order to use the cache, set the related flags in the source-controller
+Deployment config:
+
+```yaml
+ spec:
+ containers:
+ - args:
+ - --watch-all-namespaces
+ - --log-level=info
+ - --log-encoding=json
+ - --enable-leader-election
+ - --storage-path=/data
+ - --storage-adv-addr=source-controller.$(RUNTIME_NAMESPACE).svc.cluster.local.
+ ## Helm cache with up to 10 items, i.e. 10 indexes.
+ - --helm-cache-max-size=10
+ ## TTL of an index is 1 hour.
+ - --helm-cache-ttl=1h
+ ## Purge expired index every 10 minutes.
+ - --helm-cache-purge-interval=10m
+```
+
 ## HelmChart Status
 
 ### Artifact

internal/cache/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2012-2019 Patrick Mylund Nielsen and the go-cache contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.