diff --git a/controllers/helmchart_controller.go b/controllers/helmchart_controller.go index 5d4f952cd..d6c46137a 100644 --- a/controllers/helmchart_controller.go +++ b/controllers/helmchart_controller.go @@ -18,23 +18,20 @@ package controllers import ( "context" + "errors" "fmt" - "io" "net/url" "os" "path/filepath" - "regexp" + "strconv" "strings" "time" - "github.com/Masterminds/semver/v3" securejoin "github.com/cyphar/filepath-securejoin" "github.com/go-logr/logr" - helmchart "helm.sh/helm/v3/pkg/chart" - "helm.sh/helm/v3/pkg/chart/loader" - "helm.sh/helm/v3/pkg/chartutil" - "helm.sh/helm/v3/pkg/getter" + helmgetter "helm.sh/helm/v3/pkg/getter" corev1 "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -50,17 +47,17 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - "sigs.k8s.io/yaml" "github.com/fluxcd/pkg/apis/meta" "github.com/fluxcd/pkg/runtime/events" "github.com/fluxcd/pkg/runtime/metrics" "github.com/fluxcd/pkg/runtime/predicates" - "github.com/fluxcd/pkg/runtime/transform" "github.com/fluxcd/pkg/untar" sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" - "github.com/fluxcd/source-controller/internal/helm" + "github.com/fluxcd/source-controller/internal/helm/chart" + "github.com/fluxcd/source-controller/internal/helm/getter" + "github.com/fluxcd/source-controller/internal/helm/repository" ) // +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=helmcharts,verbs=get;list;watch;create;update;patch;delete @@ -73,7 +70,7 @@ type HelmChartReconciler struct { client.Client Scheme *runtime.Scheme Storage *Storage - Getters getter.Providers + Getters helmgetter.Providers EventRecorder kuberecorder.EventRecorder ExternalEventRecorder *events.Recorder MetricsRecorder *metrics.Recorder @@ -202,30 +199,28 @@ func (r *HelmChartReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{Requeue: true}, err } + // Create working directory + workDir, err := os.MkdirTemp("", chart.Kind+"-"+chart.Namespace+"-"+chart.Name+"-") + if err != nil { + err = fmt.Errorf("failed to create temporary working directory: %w", err) + chart = sourcev1.HelmChartNotReady(*chart.DeepCopy(), sourcev1.ChartPullFailedReason, err.Error()) + if err := r.updateStatus(ctx, req, chart.Status); err != nil { + log.Error(err, "unable to update status") + } + r.recordReadiness(ctx, chart) + return ctrl.Result{Requeue: true}, err + } + defer os.RemoveAll(workDir) + // Perform the reconciliation for the chart source type var reconciledChart sourcev1.HelmChart var reconcileErr error switch typedSource := source.(type) { case *sourcev1.HelmRepository: - // TODO: move this to a validation webhook once the discussion around - // certificates has settled: https://github.com/fluxcd/image-reflector-controller/issues/69 - if err := validHelmChartName(chart.Spec.Chart); err != nil { - reconciledChart = sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()) - log.Error(err, "validation failed") - if err := r.updateStatus(ctx, req, reconciledChart.Status); err != nil { - log.Info(fmt.Sprintf("%v", reconciledChart.Status)) - log.Error(err, "unable to update status") - return ctrl.Result{Requeue: true}, err - } - r.event(ctx, reconciledChart, events.EventSeverityError, err.Error()) - r.recordReadiness(ctx, reconciledChart) - // Do not requeue as there is no chance on recovery. - return ctrl.Result{Requeue: false}, nil - } - reconciledChart, reconcileErr = r.reconcileFromHelmRepository(ctx, *typedSource, *chart.DeepCopy(), changed) + reconciledChart, reconcileErr = r.fromHelmRepository(ctx, *typedSource, *chart.DeepCopy(), workDir, changed) case *sourcev1.GitRepository, *sourcev1.Bucket: - reconciledChart, reconcileErr = r.reconcileFromTarballArtifact(ctx, *typedSource.GetArtifact(), - *chart.DeepCopy(), changed) + reconciledChart, reconcileErr = r.fromTarballArtifact(ctx, *typedSource.GetArtifact(), *chart.DeepCopy(), + workDir, changed) default: err := fmt.Errorf("unable to reconcile unsupported source reference kind '%s'", chart.Spec.SourceRef.Kind) return ctrl.Result{Requeue: false}, err @@ -297,462 +292,270 @@ func (r *HelmChartReconciler) getSource(ctx context.Context, chart sourcev1.Helm return source, nil } -func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context, - repository sourcev1.HelmRepository, chart sourcev1.HelmChart, force bool) (sourcev1.HelmChart, error) { - // Configure ChartRepository getter options - clientOpts := []getter.Option{ - getter.WithURL(repository.Spec.URL), - getter.WithTimeout(repository.Spec.Timeout.Duration), - getter.WithPassCredentialsAll(repository.Spec.PassCredentials), +func (r *HelmChartReconciler) fromHelmRepository(ctx context.Context, repo sourcev1.HelmRepository, c sourcev1.HelmChart, + workDir string, force bool) (sourcev1.HelmChart, error) { + // Configure Index getter options + clientOpts := []helmgetter.Option{ + helmgetter.WithURL(repo.Spec.URL), + helmgetter.WithTimeout(repo.Spec.Timeout.Duration), + helmgetter.WithPassCredentialsAll(repo.Spec.PassCredentials), } - if secret, err := r.getHelmRepositorySecret(ctx, &repository); err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.AuthenticationFailedReason, err.Error()), err + if secret, err := r.getHelmRepositorySecret(ctx, &repo); err != nil { + return sourcev1.HelmChartNotReady(c, sourcev1.AuthenticationFailedReason, err.Error()), err } else if secret != nil { - opts, cleanup, err := helm.ClientOptionsFromSecret(*secret) + // Create temporary working directory for credentials + authDir := filepath.Join(workDir, "creds") + if err := os.Mkdir(authDir, 0700); err != nil { + err = fmt.Errorf("failed to create temporary directory for repository credentials: %w", err) + } + opts, err := getter.ClientOptionsFromSecret(authDir, *secret) if err != nil { - err = fmt.Errorf("auth options error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.AuthenticationFailedReason, err.Error()), err + err = fmt.Errorf("failed to create client options for HelmRepository '%s': %w", repo.Name, err) + return sourcev1.HelmChartNotReady(c, sourcev1.AuthenticationFailedReason, err.Error()), err } - defer cleanup() clientOpts = append(clientOpts, opts...) } - // Initialize the chart repository and load the index file - chartRepo, err := helm.NewChartRepository(repository.Spec.URL, r.Getters, clientOpts) + // Initialize the chart repository + chartRepo, err := repository.NewChartRepository(repo.Spec.URL, r.Storage.LocalPath(*repo.GetArtifact()), r.Getters, clientOpts) if err != nil { switch err.(type) { case *url.Error: - return sourcev1.HelmChartNotReady(chart, sourcev1.URLInvalidReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.URLInvalidReason, err.Error()), err default: - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.ChartPullFailedReason, err.Error()), err } } - indexFile, err := os.Open(r.Storage.LocalPath(*repository.GetArtifact())) - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + + var cachedChart string + if artifact := c.GetArtifact(); artifact != nil { + cachedChart = artifact.Path } - b, err := io.ReadAll(indexFile) - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err + + // Build the chart + cb := chart.NewRemoteBuilder(chartRepo) + ref := chart.RemoteReference{Name: c.Spec.Chart, Version: c.Spec.Version} + opts := chart.BuildOptions{ + ValuesFiles: c.GetValuesFiles(), + CachedChart: cachedChart, + Force: force, } - if err = chartRepo.LoadIndex(b); err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err + // Set the VersionMetadata to the object's Generation if ValuesFiles is defined + // This ensures changes can be noticed by the Artifact consumer + if len(opts.GetValuesFiles()) > 0 { + opts.VersionMetadata = strconv.FormatInt(c.Generation, 10) } - - // Lookup the chart version in the chart repository index - chartVer, err := chartRepo.Get(chart.Spec.Chart, chart.Spec.Version) + b, err := cb.Build(ctx, ref, filepath.Join(workDir, "chart.tgz"), opts) if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.ChartPullFailedReason, err.Error()), err } - // Return early if the revision is still the same as the current artifact - newArtifact := r.Storage.NewArtifactFor(chart.Kind, chart.GetObjectMeta(), chartVer.Version, - fmt.Sprintf("%s-%s.tgz", chartVer.Name, chartVer.Version)) - if !force && repository.GetArtifact().HasRevision(newArtifact.Revision) { - if newArtifact.URL != chart.GetArtifact().URL { - r.Storage.SetArtifactURL(chart.GetArtifact()) - chart.Status.URL = r.Storage.SetHostname(chart.Status.URL) + newArtifact := r.Storage.NewArtifactFor(c.Kind, c.GetObjectMeta(), b.Version, + fmt.Sprintf("%s-%s.tgz", b.Name, b.Version)) + + // If the path of the returned build equals the cache path, + // there are no changes to the chart + if b.Path == cachedChart { + // Ensure hostname is updated + if c.GetArtifact().URL != newArtifact.URL { + r.Storage.SetArtifactURL(c.GetArtifact()) + c.Status.URL = r.Storage.SetHostname(c.Status.URL) } - return chart, nil + return c, nil } // Ensure artifact directory exists err = r.Storage.MkdirAll(newArtifact) if err != nil { err = fmt.Errorf("unable to create chart directory: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } // Acquire a lock for the artifact unlock, err := r.Storage.Lock(newArtifact) if err != nil { err = fmt.Errorf("unable to acquire lock: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } defer unlock() - // Attempt to download the chart - res, err := chartRepo.DownloadChart(chartVer) - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err - } - tmpFile, err := os.CreateTemp("", fmt.Sprintf("%s-%s-", chart.Namespace, chart.Name)) - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err - } - defer os.RemoveAll(tmpFile.Name()) - if _, err = io.Copy(tmpFile, res); err != nil { - tmpFile.Close() - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err - } - tmpFile.Close() - - // Check if we need to repackage the chart with the declared defaults files. - var ( - pkgPath = tmpFile.Name() - readyReason = sourcev1.ChartPullSucceededReason - readyMessage = fmt.Sprintf("Fetched revision: %s", newArtifact.Revision) - ) - - switch { - case len(chart.GetValuesFiles()) > 0: - valuesMap := make(map[string]interface{}) - - // Load the chart - helmChart, err := loader.LoadFile(pkgPath) - if err != nil { - err = fmt.Errorf("load chart error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - - for _, v := range chart.GetValuesFiles() { - if v == "values.yaml" { - valuesMap = transform.MergeMaps(valuesMap, helmChart.Values) - continue - } - - var valuesData []byte - cfn := filepath.Clean(v) - for _, f := range helmChart.Files { - if f.Name == cfn { - valuesData = f.Data - break - } - } - if valuesData == nil { - err = fmt.Errorf("invalid values file path: %s", v) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - - yamlMap := make(map[string]interface{}) - err = yaml.Unmarshal(valuesData, &yamlMap) - if err != nil { - err = fmt.Errorf("unmarshaling values from %s failed: %w", v, err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - - valuesMap = transform.MergeMaps(valuesMap, yamlMap) - } - - yamlBytes, err := yaml.Marshal(valuesMap) - if err != nil { - err = fmt.Errorf("marshaling values failed: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err - } - - // Overwrite values file - if changed, err := helm.OverwriteChartDefaultValues(helmChart, yamlBytes); err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err - } else if !changed { - break - } - - // Create temporary working directory - tmpDir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s-", chart.Namespace, chart.Name)) - if err != nil { - err = fmt.Errorf("tmp dir error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - defer os.RemoveAll(tmpDir) - - // Package the chart with the new default values - pkgPath, err = chartutil.Save(helmChart, tmpDir) - if err != nil { - err = fmt.Errorf("chart package error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err - } - - // Copy the packaged chart to the artifact path - if err := r.Storage.CopyFromPath(&newArtifact, pkgPath); err != nil { - err = fmt.Errorf("failed to write chart package to storage: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - - readyMessage = fmt.Sprintf("Fetched and packaged revision: %s", newArtifact.Revision) - readyReason = sourcev1.ChartPackageSucceededReason - } - - // Write artifact to storage - if err := r.Storage.CopyFromPath(&newArtifact, pkgPath); err != nil { - err = fmt.Errorf("unable to write chart file: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + // Copy the packaged chart to the artifact path + if err = r.Storage.CopyFromPath(&newArtifact, b.Path); err != nil { + err = fmt.Errorf("failed to write chart package to storage: %w", err) + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } // Update symlink - chartUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", chartVer.Name)) + cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", b.Name)) if err != nil { err = fmt.Errorf("storage error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } - - return sourcev1.HelmChartReady(chart, newArtifact, chartUrl, readyReason, readyMessage), nil + return sourcev1.HelmChartReady(c, newArtifact, cUrl, sourcev1.ChartPullSucceededReason, b.Summary()), nil } -func (r *HelmChartReconciler) reconcileFromTarballArtifact(ctx context.Context, - artifact sourcev1.Artifact, chart sourcev1.HelmChart, force bool) (sourcev1.HelmChart, error) { - // Create temporary working directory - tmpDir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s-", chart.Namespace, chart.Name)) - if err != nil { - err = fmt.Errorf("tmp dir error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err +func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source sourcev1.Artifact, c sourcev1.HelmChart, + workDir string, force bool) (sourcev1.HelmChart, error) { + // Create temporary working directory to untar into + sourceDir := filepath.Join(workDir, "source") + if err := os.Mkdir(sourceDir, 0700); err != nil { + err = fmt.Errorf("failed to create temporary directory to untar source into: %w", err) + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } - defer os.RemoveAll(tmpDir) // Open the tarball artifact file and untar files into working directory - f, err := os.Open(r.Storage.LocalPath(artifact)) + f, err := os.Open(r.Storage.LocalPath(source)) if err != nil { err = fmt.Errorf("artifact open error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } - if _, err = untar.Untar(f, tmpDir); err != nil { - f.Close() + if _, err = untar.Untar(f, sourceDir); err != nil { + _ = f.Close() err = fmt.Errorf("artifact untar error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } - f.Close() - - // Load the chart - chartPath, err := securejoin.SecureJoin(tmpDir, chart.Spec.Chart) - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + if err = f.Close(); err != nil { + err = fmt.Errorf("artifact close error: %w", err) + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } - chartFileInfo, err := os.Stat(chartPath) + + chartPath, err := securejoin.SecureJoin(sourceDir, c.Spec.Chart) if err != nil { - err = fmt.Errorf("chart location read error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } - helmChart, err := loader.Load(chartPath) - if err != nil { - err = fmt.Errorf("load chart error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + + // Setup dependency manager + authDir := filepath.Join(workDir, "creds") + if err = os.Mkdir(authDir, 0700); err != nil { + err = fmt.Errorf("failed to create temporaRy directory for dependency credentials: %w", err) + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } + dm := chart.NewDependencyManager( + chart.WithRepositoryCallback(r.namespacedChartRepositoryCallback(ctx, authDir, c.GetNamespace())), + ) + defer dm.Clear() - v, err := semver.NewVersion(helmChart.Metadata.Version) - if err != nil { - err = fmt.Errorf("semver parse error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + // Configure builder options, including any previously cached chart + opts := chart.BuildOptions{ + ValuesFiles: c.GetValuesFiles(), + Force: force, + } + if artifact := c.Status.Artifact; artifact != nil { + opts.CachedChart = artifact.Path } - version := v.String() - if chart.Spec.ReconcileStrategy == sourcev1.ReconcileStrategyRevision { + // Add revision metadata to chart build + if c.Spec.ReconcileStrategy == sourcev1.ReconcileStrategyRevision { // Isolate the commit SHA from GitRepository type artifacts by removing the branch/ prefix. - splitRev := strings.Split(artifact.Revision, "/") - v, err := v.SetMetadata(splitRev[len(splitRev)-1]) - if err != nil { - err = fmt.Errorf("semver parse error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - - version = v.String() - helmChart.Metadata.Version = v.String() + splitRev := strings.Split(source.Revision, "/") + opts.VersionMetadata = splitRev[len(splitRev)-1] } - - // Return early if the revision is still the same as the current chart artifact - newArtifact := r.Storage.NewArtifactFor(chart.Kind, chart.ObjectMeta.GetObjectMeta(), version, - fmt.Sprintf("%s-%s.tgz", helmChart.Metadata.Name, version)) - if !force && apimeta.IsStatusConditionTrue(chart.Status.Conditions, meta.ReadyCondition) && chart.GetArtifact().HasRevision(newArtifact.Revision) { - if newArtifact.URL != artifact.URL { - r.Storage.SetArtifactURL(chart.GetArtifact()) - chart.Status.URL = r.Storage.SetHostname(chart.Status.URL) + // Set the VersionMetadata to the object's Generation if ValuesFiles is defined + // This ensures changes can be noticed by the Artifact consumer + if len(opts.GetValuesFiles()) > 0 { + if opts.VersionMetadata != "" { + opts.VersionMetadata += "." } - return chart, nil + opts.VersionMetadata += strconv.FormatInt(c.Generation, 10) } - // Either (re)package the chart with the declared default values file, - // or write the chart directly to storage. - pkgPath := chartPath - isValuesFileOverriden := false - if len(chart.GetValuesFiles()) > 0 { - valuesMap := make(map[string]interface{}) - for _, v := range chart.GetValuesFiles() { - srcPath, err := securejoin.SecureJoin(tmpDir, v) - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - if f, err := os.Stat(srcPath); os.IsNotExist(err) || !f.Mode().IsRegular() { - err = fmt.Errorf("invalid values file path: %s", v) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - - valuesData, err := os.ReadFile(srcPath) - if err != nil { - err = fmt.Errorf("failed to read from values file '%s': %w", v, err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - - yamlMap := make(map[string]interface{}) - err = yaml.Unmarshal(valuesData, &yamlMap) - if err != nil { - err = fmt.Errorf("unmarshaling values from %s failed: %w", v, err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - - valuesMap = transform.MergeMaps(valuesMap, yamlMap) - } - - yamlBytes, err := yaml.Marshal(valuesMap) - if err != nil { - err = fmt.Errorf("marshaling values failed: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err - } - - isValuesFileOverriden, err = helm.OverwriteChartDefaultValues(helmChart, yamlBytes) - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err - } + // Build chart + cb := chart.NewLocalBuilder(dm) + b, err := cb.Build(ctx, chart.LocalReference{WorkDir: sourceDir, Path: chartPath}, filepath.Join(workDir, "chart.tgz"), opts) + if err != nil { + return sourcev1.HelmChartNotReady(c, reasonForBuildError(err), err.Error()), err } - isDir := chartFileInfo.IsDir() - switch { - case isDir: - // Determine chart dependencies - deps := helmChart.Dependencies() - reqs := helmChart.Metadata.Dependencies - lock := helmChart.Lock - if lock != nil { - // Load from lockfile if exists - reqs = lock.Dependencies - } - var dwr []*helm.DependencyWithRepository - for _, dep := range reqs { - // Exclude existing dependencies - found := false - for _, existing := range deps { - if existing.Name() == dep.Name { - found = true - } - } - if found { - continue - } - - // Continue loop if file scheme detected - if dep.Repository == "" || strings.HasPrefix(dep.Repository, "file://") { - dwr = append(dwr, &helm.DependencyWithRepository{ - Dependency: dep, - Repository: nil, - }) - continue - } - - // Discover existing HelmRepository by URL - repository, err := r.resolveDependencyRepository(ctx, dep, chart.Namespace) - if err != nil { - repository = &sourcev1.HelmRepository{ - Spec: sourcev1.HelmRepositorySpec{ - URL: dep.Repository, - Timeout: &metav1.Duration{Duration: 60 * time.Second}, - }, - } - } - - // Configure ChartRepository getter options - clientOpts := []getter.Option{ - getter.WithURL(repository.Spec.URL), - getter.WithTimeout(repository.Spec.Timeout.Duration), - getter.WithPassCredentialsAll(repository.Spec.PassCredentials), - } - if secret, err := r.getHelmRepositorySecret(ctx, repository); err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.AuthenticationFailedReason, err.Error()), err - } else if secret != nil { - opts, cleanup, err := helm.ClientOptionsFromSecret(*secret) - if err != nil { - err = fmt.Errorf("auth options error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.AuthenticationFailedReason, err.Error()), err - } - defer cleanup() - clientOpts = append(clientOpts, opts...) - } - - // Initialize the chart repository and load the index file - chartRepo, err := helm.NewChartRepository(repository.Spec.URL, r.Getters, clientOpts) - if err != nil { - switch err.(type) { - case *url.Error: - return sourcev1.HelmChartNotReady(chart, sourcev1.URLInvalidReason, err.Error()), err - default: - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err - } - } - if repository.Status.Artifact != nil { - indexFile, err := os.Open(r.Storage.LocalPath(*repository.GetArtifact())) - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - b, err := io.ReadAll(indexFile) - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err - } - if err = chartRepo.LoadIndex(b); err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err - } - } else { - // Download index - err = chartRepo.DownloadIndex() - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err - } - } - - dwr = append(dwr, &helm.DependencyWithRepository{ - Dependency: dep, - Repository: chartRepo, - }) - } - - // Construct dependencies for chart if any - if len(dwr) > 0 { - dm := &helm.DependencyManager{ - WorkingDir: tmpDir, - ChartPath: chart.Spec.Chart, - Chart: helmChart, - Dependencies: dwr, - } - err = dm.Build(ctx) - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - } + newArtifact := r.Storage.NewArtifactFor(c.Kind, c.GetObjectMeta(), b.Version, + fmt.Sprintf("%s-%s.tgz", b.Name, b.Version)) - fallthrough - case isValuesFileOverriden: - pkgPath, err = chartutil.Save(helmChart, tmpDir) - if err != nil { - err = fmt.Errorf("chart package error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err + // If the path of the returned build equals the cache path, + // there are no changes to the chart + if apimeta.IsStatusConditionTrue(c.Status.Conditions, meta.ReadyCondition) && + b.Path == opts.CachedChart { + // Ensure hostname is updated + if c.GetArtifact().URL != newArtifact.URL { + r.Storage.SetArtifactURL(c.GetArtifact()) + c.Status.URL = r.Storage.SetHostname(c.Status.URL) } + return c, nil } // Ensure artifact directory exists err = r.Storage.MkdirAll(newArtifact) if err != nil { - err = fmt.Errorf("unable to create artifact directory: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + err = fmt.Errorf("unable to create chart directory: %w", err) + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } // Acquire a lock for the artifact unlock, err := r.Storage.Lock(newArtifact) if err != nil { err = fmt.Errorf("unable to acquire lock: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } defer unlock() // Copy the packaged chart to the artifact path - if err := r.Storage.CopyFromPath(&newArtifact, pkgPath); err != nil { + if err = r.Storage.CopyFromPath(&newArtifact, b.Path); err != nil { err = fmt.Errorf("failed to write chart package to storage: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } // Update symlink - cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", helmChart.Metadata.Name)) + cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", b.Name)) if err != nil { err = fmt.Errorf("storage error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } - message := fmt.Sprintf("Fetched and packaged revision: %s", newArtifact.Revision) - return sourcev1.HelmChartReady(chart, newArtifact, cUrl, sourcev1.ChartPackageSucceededReason, message), nil + return sourcev1.HelmChartReady(c, newArtifact, cUrl, reasonForBuildSuccess(b), b.Summary()), nil +} + +// namespacedChartRepositoryCallback returns a chart.GetChartRepositoryCallback +// scoped to the given namespace. Credentials for retrieved v1beta1.HelmRepository +// objects are stored in the given directory. +// The returned callback returns a repository.ChartRepository configured with the +// retrieved v1beta1.HelmRepository, or a shim with defaults if no object could +// be found. +func (r *HelmChartReconciler) namespacedChartRepositoryCallback(ctx context.Context, dir, namespace string) chart.GetChartRepositoryCallback { + return func(url string) (*repository.ChartRepository, error) { + repo, err := r.resolveDependencyRepository(ctx, url, namespace) + if err != nil { + // Return Kubernetes client errors, but ignore others + if apierrs.ReasonForError(err) != metav1.StatusReasonUnknown { + return nil, err + } + repo = &sourcev1.HelmRepository{ + Spec: sourcev1.HelmRepositorySpec{ + URL: url, + Timeout: &metav1.Duration{Duration: 60 * time.Second}, + }, + } + } + clientOpts := []helmgetter.Option{ + helmgetter.WithURL(repo.Spec.URL), + helmgetter.WithTimeout(repo.Spec.Timeout.Duration), + helmgetter.WithPassCredentialsAll(repo.Spec.PassCredentials), + } + if secret, err := r.getHelmRepositorySecret(ctx, repo); err != nil { + return nil, err + } else if secret != nil { + opts, err := getter.ClientOptionsFromSecret(dir, *secret) + if err != nil { + return nil, err + } + clientOpts = append(clientOpts, opts...) + } + chartRepo, err := repository.NewChartRepository(repo.Spec.URL, "", r.Getters, clientOpts) + if err != nil { + return nil, err + } + if repo.Status.Artifact != nil { + chartRepo.CachePath = r.Storage.LocalPath(*repo.GetArtifact()) + } + return chartRepo, nil + } } func (r *HelmChartReconciler) reconcileDelete(ctx context.Context, chart sourcev1.HelmChart) (ctrl.Result, error) { @@ -865,7 +668,7 @@ func (r *HelmChartReconciler) indexHelmRepositoryByURL(o client.Object) []string if !ok { panic(fmt.Sprintf("Expected a HelmRepository, got %T", o)) } - u := helm.NormalizeChartRepositoryURL(repo.Spec.URL) + u := repository.NormalizeURL(repo.Spec.URL) if u != "" { return []string{u} } @@ -880,15 +683,10 @@ func (r *HelmChartReconciler) indexHelmChartBySource(o client.Object) []string { return []string{fmt.Sprintf("%s/%s", hc.Spec.SourceRef.Kind, hc.Spec.SourceRef.Name)} } -func (r *HelmChartReconciler) resolveDependencyRepository(ctx context.Context, dep *helmchart.Dependency, namespace string) (*sourcev1.HelmRepository, error) { - u := helm.NormalizeChartRepositoryURL(dep.Repository) - if u == "" { - return nil, fmt.Errorf("invalid repository URL") - } - +func (r *HelmChartReconciler) resolveDependencyRepository(ctx context.Context, url string, namespace string) (*sourcev1.HelmRepository, error) { listOpts := []client.ListOption{ client.InNamespace(namespace), - client.MatchingFields{sourcev1.HelmRepositoryURLIndexKey: u}, + client.MatchingFields{sourcev1.HelmRepositoryURLIndexKey: url}, } var list sourcev1.HelmRepositoryList err := r.Client.List(ctx, &list, listOpts...) @@ -898,8 +696,7 @@ func (r *HelmChartReconciler) resolveDependencyRepository(ctx context.Context, d if len(list.Items) > 0 { return &list.Items[0], nil } - - return nil, fmt.Errorf("no HelmRepository found") + return nil, fmt.Errorf("no HelmRepository found for '%s' in '%s' namespace", url, namespace) } func (r *HelmChartReconciler) getHelmRepositorySecret(ctx context.Context, repository *sourcev1.HelmRepository) (*corev1.Secret, error) { @@ -917,7 +714,6 @@ func (r *HelmChartReconciler) getHelmRepositorySecret(ctx context.Context, repos } return &secret, nil } - return nil, nil } @@ -1008,18 +804,6 @@ func (r *HelmChartReconciler) requestsForBucketChange(o client.Object) []reconci return reqs } -// validHelmChartName returns an error if the given string is not a -// valid Helm chart name; a valid name must be lower case letters -// and numbers, words may be separated with dashes (-). -// Ref: https://helm.sh/docs/chart_best_practices/conventions/#chart-names -func validHelmChartName(s string) error { - chartFmt := regexp.MustCompile("^([-a-z0-9]*)$") - if !chartFmt.MatchString(s) { - return fmt.Errorf("invalid chart name %q, a valid name must be lower case letters and numbers and MAY be separated with dashes (-)", s) - } - return nil -} - func (r *HelmChartReconciler) recordSuspension(ctx context.Context, chart sourcev1.HelmChart) { if r.MetricsRecorder == nil { return @@ -1038,3 +822,23 @@ func (r *HelmChartReconciler) recordSuspension(ctx context.Context, chart source r.MetricsRecorder.RecordSuspend(*objRef, chart.Spec.Suspend) } } + +func reasonForBuildError(err error) string { + var buildErr *chart.BuildError + if ok := errors.As(err, &buildErr); !ok { + return sourcev1.ChartPullFailedReason + } + switch buildErr.Reason { + case chart.ErrChartMetadataPatch, chart.ErrValuesFilesMerge, chart.ErrDependencyBuild, chart.ErrChartPackage: + return sourcev1.ChartPackageFailedReason + default: + return sourcev1.ChartPullFailedReason + } +} + +func reasonForBuildSuccess(result *chart.Build) string { + if result.Packaged { + return sourcev1.ChartPackageSucceededReason + } + return sourcev1.ChartPullSucceededReason +} diff --git a/controllers/helmchart_controller_test.go b/controllers/helmchart_controller_test.go index 35462d467..82df1bc35 100644 --- a/controllers/helmchart_controller_test.go +++ b/controllers/helmchart_controller_test.go @@ -25,7 +25,6 @@ import ( "path" "path/filepath" "strings" - "testing" "time" "github.com/fluxcd/pkg/apis/meta" @@ -732,6 +731,7 @@ var _ = Describe("HelmChartReconciler", func() { }, timeout, interval).Should(BeTrue()) helmChart, err := loader.Load(storage.LocalPath(*now.Status.Artifact)) Expect(err).NotTo(HaveOccurred()) + Expect(helmChart.Values).ToNot(BeNil()) Expect(helmChart.Values["testDefault"]).To(BeTrue()) Expect(helmChart.Values["testOverride"]).To(BeFalse()) @@ -1326,26 +1326,3 @@ var _ = Describe("HelmChartReconciler", func() { }) }) }) - -func Test_validHelmChartName(t *testing.T) { - tests := []struct { - name string - chart string - expectErr bool - }{ - {"valid", "drupal", false}, - {"valid dash", "nginx-lego", false}, - {"valid dashes", "aws-cluster-autoscaler", false}, - {"valid alphanum", "ng1nx-leg0", false}, - {"invalid slash", "artifactory/invalid", true}, - {"invalid dot", "in.valid", true}, - {"invalid uppercase", "inValid", true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := validHelmChartName(tt.chart); (err != nil) != tt.expectErr { - t.Errorf("validHelmChartName() error = %v, expectErr %v", err, tt.expectErr) - } - }) - } -} diff --git a/controllers/helmrepository_controller.go b/controllers/helmrepository_controller.go index b7f8cd516..5a29a7734 100644 --- a/controllers/helmrepository_controller.go +++ b/controllers/helmrepository_controller.go @@ -17,14 +17,14 @@ limitations under the License. package controllers import ( - "bytes" "context" "fmt" "net/url" + "os" "time" "github.com/go-logr/logr" - "helm.sh/helm/v3/pkg/getter" + helmgetter "helm.sh/helm/v3/pkg/getter" corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -37,7 +37,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/yaml" "github.com/fluxcd/pkg/apis/meta" "github.com/fluxcd/pkg/runtime/events" @@ -45,7 +44,8 @@ import ( "github.com/fluxcd/pkg/runtime/predicates" sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" - "github.com/fluxcd/source-controller/internal/helm" + "github.com/fluxcd/source-controller/internal/helm/getter" + "github.com/fluxcd/source-controller/internal/helm/repository" ) // +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=helmrepositories,verbs=get;list;watch;create;update;patch;delete @@ -58,7 +58,7 @@ type HelmRepositoryReconciler struct { client.Client Scheme *runtime.Scheme Storage *Storage - Getters getter.Providers + Getters helmgetter.Providers EventRecorder kuberecorder.EventRecorder ExternalEventRecorder *events.Recorder MetricsRecorder *metrics.Recorder @@ -170,96 +170,108 @@ func (r *HelmRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reque return ctrl.Result{RequeueAfter: repository.GetInterval().Duration}, nil } -func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sourcev1.HelmRepository) (sourcev1.HelmRepository, error) { - clientOpts := []getter.Option{ - getter.WithURL(repository.Spec.URL), - getter.WithTimeout(repository.Spec.Timeout.Duration), - getter.WithPassCredentialsAll(repository.Spec.PassCredentials), +func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repo sourcev1.HelmRepository) (sourcev1.HelmRepository, error) { + clientOpts := []helmgetter.Option{ + helmgetter.WithURL(repo.Spec.URL), + helmgetter.WithTimeout(repo.Spec.Timeout.Duration), + helmgetter.WithPassCredentialsAll(repo.Spec.PassCredentials), } - if repository.Spec.SecretRef != nil { + if repo.Spec.SecretRef != nil { name := types.NamespacedName{ - Namespace: repository.GetNamespace(), - Name: repository.Spec.SecretRef.Name, + Namespace: repo.GetNamespace(), + Name: repo.Spec.SecretRef.Name, } var secret corev1.Secret err := r.Client.Get(ctx, name, &secret) if err != nil { err = fmt.Errorf("auth secret error: %w", err) - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.AuthenticationFailedReason, err.Error()), err } - opts, cleanup, err := helm.ClientOptionsFromSecret(secret) + authDir, err := os.MkdirTemp("", repo.Kind+"-"+repo.Namespace+"-"+repo.Name+"-") + if err != nil { + err = fmt.Errorf("failed to create temporary working directory for credentials: %w", err) + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.AuthenticationFailedReason, err.Error()), err + } + defer os.RemoveAll(authDir) + + opts, err := getter.ClientOptionsFromSecret(authDir, secret) if err != nil { err = fmt.Errorf("auth options error: %w", err) - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.AuthenticationFailedReason, err.Error()), err } - defer cleanup() clientOpts = append(clientOpts, opts...) } - chartRepo, err := helm.NewChartRepository(repository.Spec.URL, r.Getters, clientOpts) + chartRepo, err := repository.NewChartRepository(repo.Spec.URL, "", r.Getters, clientOpts) if err != nil { switch err.(type) { case *url.Error: - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.URLInvalidReason, err.Error()), err + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.URLInvalidReason, err.Error()), err default: - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.IndexationFailedReason, err.Error()), err + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.IndexationFailedReason, err.Error()), err } } - if err := chartRepo.DownloadIndex(); err != nil { + checksum, err := chartRepo.CacheIndex() + if err != nil { err = fmt.Errorf("failed to download repository index: %w", err) - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.IndexationFailedReason, err.Error()), err + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.IndexationFailedReason, err.Error()), err } + defer chartRepo.RemoveCache() - indexBytes, err := yaml.Marshal(&chartRepo.Index) - if err != nil { - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err - } - hash := r.Storage.Checksum(bytes.NewReader(indexBytes)) - artifact := r.Storage.NewArtifactFor(repository.Kind, - repository.ObjectMeta.GetObjectMeta(), - hash, - fmt.Sprintf("index-%s.yaml", hash)) - // return early on unchanged index - if apimeta.IsStatusConditionTrue(repository.Status.Conditions, meta.ReadyCondition) && repository.GetArtifact().HasRevision(artifact.Revision) { - if artifact.URL != repository.GetArtifact().URL { - r.Storage.SetArtifactURL(repository.GetArtifact()) - repository.Status.URL = r.Storage.SetHostname(repository.Status.URL) + artifact := r.Storage.NewArtifactFor(repo.Kind, + repo.ObjectMeta.GetObjectMeta(), + "", + fmt.Sprintf("index-%s.yaml", checksum)) + + // Return early on unchanged index + if apimeta.IsStatusConditionTrue(repo.Status.Conditions, meta.ReadyCondition) && + (repo.GetArtifact() != nil && repo.GetArtifact().Checksum == checksum) { + if artifact.URL != repo.GetArtifact().URL { + r.Storage.SetArtifactURL(repo.GetArtifact()) + repo.Status.URL = r.Storage.SetHostname(repo.Status.URL) } - return repository, nil + return repo, nil + } + + // Load the cached repository index to ensure it passes validation + if err := chartRepo.LoadFromCache(); err != nil { + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.IndexationFailedReason, err.Error()), err } + // The repository checksum is the SHA256 of the loaded bytes, after sorting + artifact.Revision = chartRepo.Checksum + chartRepo.Unload() - // create artifact dir + // Create artifact dir err = r.Storage.MkdirAll(artifact) if err != nil { err = fmt.Errorf("unable to create repository index directory: %w", err) - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.StorageOperationFailedReason, err.Error()), err } - // acquire lock + // Acquire lock unlock, err := r.Storage.Lock(artifact) if err != nil { err = fmt.Errorf("unable to acquire lock: %w", err) - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.StorageOperationFailedReason, err.Error()), err } defer unlock() - // save artifact to storage - if err := r.Storage.AtomicWriteFile(&artifact, bytes.NewReader(indexBytes), 0644); err != nil { - err = fmt.Errorf("unable to write repository index file: %w", err) - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err + // Save artifact to storage + if err = r.Storage.CopyFromPath(&artifact, chartRepo.CachePath); err != nil { + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.StorageOperationFailedReason, err.Error()), err } - // update index symlink + // Update index symlink indexURL, err := r.Storage.Symlink(artifact, "index.yaml") if err != nil { err = fmt.Errorf("storage error: %w", err) - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.StorageOperationFailedReason, err.Error()), err } message := fmt.Sprintf("Fetched revision: %s", artifact.Revision) - return sourcev1.HelmRepositoryReady(repository, artifact, indexURL, sourcev1.IndexationSucceededReason, message), nil + return sourcev1.HelmRepositoryReady(repo, artifact, indexURL, sourcev1.IndexationSucceededReason, message), nil } func (r *HelmRepositoryReconciler) reconcileDelete(ctx context.Context, repository sourcev1.HelmRepository) (ctrl.Result, error) { diff --git a/go.mod b/go.mod index c4503b710..5246fc455 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/minio/minio-go/v7 v7.0.10 github.com/onsi/ginkgo v1.16.4 github.com/onsi/gomega v1.14.0 + github.com/otiai10/copy v1.7.0 github.com/spf13/pflag v1.0.5 github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940 // indirect github.com/yvasiyarov/gorelic v0.0.7 // indirect diff --git a/go.sum b/go.sum index 593aa3e0b..a252cf16f 100644 --- a/go.sum +++ b/go.sum @@ -738,6 +738,13 @@ github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxS github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/otiai10/copy v1.7.0 h1:hVoPiN+t+7d2nzzwMiDHPSOogsWAStewq3TwU05+clE= +github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= +github.com/otiai10/mint v1.3.3 h1:7JgpsBaN0uMkyju4tbYHu0mnM55hNKVYLsXmwr15NQI= +github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= diff --git a/internal/helm/chart.go b/internal/helm/chart.go deleted file mode 100644 index 6630f4f74..000000000 --- a/internal/helm/chart.go +++ /dev/null @@ -1,59 +0,0 @@ -/* -Copyright 2020 The Flux authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package helm - -import ( - "fmt" - "reflect" - - helmchart "helm.sh/helm/v3/pkg/chart" - "helm.sh/helm/v3/pkg/chartutil" -) - -// OverwriteChartDefaultValues overwrites the chart default values file with the -// given data. -func OverwriteChartDefaultValues(chart *helmchart.Chart, data []byte) (bool, error) { - // Read override values file data - values, err := chartutil.ReadValues(data) - if err != nil { - return false, fmt.Errorf("failed to parse provided override values file data") - } - - // Replace current values file in Raw field - for _, f := range chart.Raw { - if f.Name == chartutil.ValuesfileName { - // Do nothing if contents are equal - if reflect.DeepEqual(f.Data, data) { - return false, nil - } - - // Replace in Files field - for _, f := range chart.Files { - if f.Name == chartutil.ValuesfileName { - f.Data = data - } - } - - f.Data = data - chart.Values = values - return true, nil - } - } - - // This should never happen, helm charts must have a values.yaml file to be valid - return false, fmt.Errorf("failed to locate values file: %s", chartutil.ValuesfileName) -} diff --git a/internal/helm/chart/builder.go b/internal/helm/chart/builder.go new file mode 100644 index 000000000..9aa2a17e4 --- /dev/null +++ b/internal/helm/chart/builder.go @@ -0,0 +1,189 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package chart + +import ( + "context" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + helmchart "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chartutil" + + "github.com/fluxcd/source-controller/internal/fs" +) + +// Reference holds information to locate a chart. +type Reference interface { + // Validate returns an error if the Reference is not valid according + // to the spec of the interface implementation. + Validate() error +} + +// LocalReference contains sufficient information to locate a chart on the +// local filesystem. +type LocalReference struct { + // WorkDir used as chroot during build operations. + // File references are not allowed to traverse outside it. + WorkDir string + // Path of the chart on the local filesystem. + Path string +} + +// Validate returns an error if the LocalReference does not have +// a Path set. +func (r LocalReference) Validate() error { + if r.Path == "" { + return fmt.Errorf("no path set for local chart reference") + } + return nil +} + +// RemoteReference contains sufficient information to look up a chart in +// a ChartRepository. +type RemoteReference struct { + // Name of the chart. + Name string + // Version of the chart. + // Can be a Semver range, or empty for latest. + Version string +} + +// Validate returns an error if the RemoteReference does not have +// a Name set. +func (r RemoteReference) Validate() error { + if r.Name == "" { + return fmt.Errorf("no name set for remote chart reference") + } + name := regexp.MustCompile("^([-a-z0-9]*)$") + if !name.MatchString(r.Name) { + return fmt.Errorf("invalid chart name '%s': a valid name must be lower case letters and numbers and MAY be separated with dashes (-)", r.Name) + } + return nil +} + +// Builder is capable of building a (specific) chart Reference. +type Builder interface { + // Build pulls and (optionally) packages a Helm chart with the given + // Reference and BuildOptions, and writes it to p. + // It returns the Build result, or an error. + // It may return an error for unsupported Reference implementations. + Build(ctx context.Context, ref Reference, p string, opts BuildOptions) (*Build, error) +} + +// BuildOptions provides a list of options for Builder.Build. +type BuildOptions struct { + // VersionMetadata can be set to SemVer build metadata as defined in + // the spec, and is included during packaging. + // Ref: https://semver.org/#spec-item-10 + VersionMetadata string + // ValuesFiles can be set to a list of relative paths, used to compose + // and overwrite an alternative default "values.yaml" for the chart. + ValuesFiles []string + // CachedChart can be set to the absolute path of a chart stored on + // the local filesystem, and is used for simple validation by metadata + // comparisons. + CachedChart string + // Force can be set to force the build of the chart, for example + // because the list of ValuesFiles has changed. + Force bool +} + +// GetValuesFiles returns BuildOptions.ValuesFiles, except if it equals +// "values.yaml", which returns nil. +func (o BuildOptions) GetValuesFiles() []string { + if len(o.ValuesFiles) == 1 && filepath.Clean(o.ValuesFiles[0]) == filepath.Clean(chartutil.ValuesfileName) { + return nil + } + return o.ValuesFiles +} + +// Build contains the Builder.Build result, including specific +// information about the built chart like ResolvedDependencies. +type Build struct { + // Path is the absolute path to the packaged chart. + Path string + // Name of the packaged chart. + Name string + // Version of the packaged chart. + Version string + // ValuesFiles is the list of files used to compose the chart's + // default "values.yaml". + ValuesFiles []string + // ResolvedDependencies is the number of local and remote dependencies + // collected by the DependencyManager before building the chart. + ResolvedDependencies int + // Packaged indicates if the Builder has packaged the chart. + // This can for example be false if ValuesFiles is empty and the chart + // source was already packaged. + Packaged bool +} + +// Summary returns a human-readable summary of the Build. +func (b *Build) Summary() string { + if b == nil || b.Name == "" || b.Version == "" { + return "No chart build." + } + + var s strings.Builder + + var action = "Pulled" + if b.Packaged { + action = "Packaged" + } + s.WriteString(fmt.Sprintf("%s '%s' chart with version '%s'", action, b.Name, b.Version)) + + if b.Packaged && len(b.ValuesFiles) > 0 { + s.WriteString(fmt.Sprintf(", with merged values files %v", b.ValuesFiles)) + } + + if b.Packaged && b.ResolvedDependencies > 0 { + s.WriteString(fmt.Sprintf(", resolving %d dependencies before packaging", b.ResolvedDependencies)) + } + + s.WriteString(".") + return s.String() +} + +// String returns the Path of the Build. +func (b *Build) String() string { + if b == nil { + return "" + } + return b.Path +} + +// packageToPath attempts to package the given chart to the out filepath. +func packageToPath(chart *helmchart.Chart, out string) error { + o, err := os.MkdirTemp("", "chart-build-*") + if err != nil { + return fmt.Errorf("failed to create temporary directory for chart: %w", err) + } + defer os.RemoveAll(o) + + p, err := chartutil.Save(chart, o) + if err != nil { + return fmt.Errorf("failed to package chart: %w", err) + } + if err = fs.RenameWithFallback(p, out); err != nil { + return fmt.Errorf("failed to write chart to file: %w", err) + } + return nil +} diff --git a/internal/helm/chart/builder_local.go b/internal/helm/chart/builder_local.go new file mode 100644 index 000000000..963588815 --- /dev/null +++ b/internal/helm/chart/builder_local.go @@ -0,0 +1,216 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package chart + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/Masterminds/semver/v3" + securejoin "github.com/cyphar/filepath-securejoin" + "helm.sh/helm/v3/pkg/chart/loader" + "sigs.k8s.io/yaml" + + "github.com/fluxcd/pkg/runtime/transform" +) + +type localChartBuilder struct { + dm *DependencyManager +} + +// NewLocalBuilder returns a Builder capable of building a Helm chart with a +// LocalReference. For chart references pointing to a directory, the +// DependencyManager is used to resolve missing local and remote dependencies. +func NewLocalBuilder(dm *DependencyManager) Builder { + return &localChartBuilder{ + dm: dm, + } +} + +// Build attempts to build a Helm chart with the given LocalReference and +// BuildOptions, writing it to p. +// It returns a Build describing the produced (or from cache observed) chart +// written to p, or a BuildError. +// +// The chart is loaded from the LocalReference.Path, and only packaged if the +// version (including BuildOptions.VersionMetadata modifications) differs from +// the current BuildOptions.CachedChart. +// +// BuildOptions.ValuesFiles changes are in this case not taken into account, +// and BuildOptions.Force should be used to enforce a rebuild. +// +// If the LocalReference.Path refers to an already packaged chart, and no +// packaging is required due to BuildOptions modifying the chart, +// LocalReference.Path is copied to p. +// +// If the LocalReference.Path refers to a chart directory, dependencies are +// confirmed to be present using the DependencyManager, while attempting to +// resolve any missing. +func (b *localChartBuilder) Build(ctx context.Context, ref Reference, p string, opts BuildOptions) (*Build, error) { + localRef, ok := ref.(LocalReference) + if !ok { + err := fmt.Errorf("expected local chart reference") + return nil, &BuildError{Reason: ErrChartReference, Err: err} + } + + if err := ref.Validate(); err != nil { + return nil, &BuildError{Reason: ErrChartReference, Err: err} + } + + // Load the chart metadata from the LocalReference to ensure it points + // to a chart + curMeta, err := LoadChartMetadata(localRef.Path) + if err != nil { + return nil, &BuildError{Reason: ErrChartPull, Err: err} + } + + result := &Build{} + result.Name = curMeta.Name + + // Set build specific metadata if instructed + result.Version = curMeta.Version + if opts.VersionMetadata != "" { + ver, err := semver.NewVersion(curMeta.Version) + if err != nil { + err = fmt.Errorf("failed to parse version from chart metadata as SemVer: %w", err) + return nil, &BuildError{Reason: ErrChartMetadataPatch, Err: err} + } + if *ver, err = ver.SetMetadata(opts.VersionMetadata); err != nil { + err = fmt.Errorf("failed to set SemVer metadata on chart version: %w", err) + return nil, &BuildError{Reason: ErrChartMetadataPatch, Err: err} + } + result.Version = ver.String() + } + + // If all the following is true, we do not need to package the chart: + // - Chart name from cached chart matches resolved name + // - Chart version from cached chart matches calculated version + // - BuildOptions.Force is False + if opts.CachedChart != "" && !opts.Force { + if curMeta, err = LoadChartMetadataFromArchive(opts.CachedChart); err == nil { + if result.Name == curMeta.Name && result.Version == curMeta.Version { + result.Path = opts.CachedChart + result.ValuesFiles = opts.ValuesFiles + return result, nil + } + } + } + + // If the chart at the path is already packaged and no custom values files + // options are set, we can copy the chart without making modifications + isChartDir := pathIsDir(localRef.Path) + if !isChartDir && len(opts.GetValuesFiles()) == 0 { + if err = copyFileToPath(localRef.Path, p); err != nil { + return nil, &BuildError{Reason: ErrChartPull, Err: err} + } + result.Path = p + return result, nil + } + + // Merge chart values, if instructed + var mergedValues map[string]interface{} + if len(opts.GetValuesFiles()) > 0 { + if mergedValues, err = mergeFileValues(localRef.WorkDir, opts.ValuesFiles); err != nil { + return nil, &BuildError{Reason: ErrValuesFilesMerge, Err: err} + } + } + + // At this point we are certain we need to load the chart; + // either to package it because it originates from a directory, + // or because we have merged values and need to repackage + chart, err := loader.Load(localRef.Path) + if err != nil { + return nil, &BuildError{Reason: ErrChartPackage, Err: err} + } + // Set earlier resolved version (with metadata) + chart.Metadata.Version = result.Version + + // Overwrite default values with merged values, if any + if ok, err = OverwriteChartDefaultValues(chart, mergedValues); ok || err != nil { + if err != nil { + return nil, &BuildError{Reason: ErrValuesFilesMerge, Err: err} + } + result.ValuesFiles = opts.GetValuesFiles() + } + + // Ensure dependencies are fetched if building from a directory + if isChartDir { + if b.dm == nil { + err = fmt.Errorf("local chart builder requires dependency manager for unpackaged charts") + return nil, &BuildError{Reason: ErrDependencyBuild, Err: err} + } + if result.ResolvedDependencies, err = b.dm.Build(ctx, ref, chart); err != nil { + return nil, &BuildError{Reason: ErrDependencyBuild, Err: err} + } + } + + // Package the chart + if err = packageToPath(chart, p); err != nil { + return nil, &BuildError{Reason: ErrChartPackage, Err: err} + } + result.Path = p + result.Packaged = true + return result, nil +} + +// mergeFileValues merges the given value file paths into a single "values.yaml" map. +// The provided (relative) paths may not traverse outside baseDir. It returns the merge +// result, or an error. +func mergeFileValues(baseDir string, paths []string) (map[string]interface{}, error) { + mergedValues := make(map[string]interface{}) + for _, p := range paths { + secureP, err := securejoin.SecureJoin(baseDir, p) + if err != nil { + return nil, err + } + if f, err := os.Stat(secureP); os.IsNotExist(err) || !f.Mode().IsRegular() { + return nil, fmt.Errorf("no values file found at path '%s' (reference '%s')", + strings.TrimPrefix(secureP, baseDir), p) + } + b, err := os.ReadFile(secureP) + if err != nil { + return nil, fmt.Errorf("could not read values from file '%s': %w", p, err) + } + values := make(map[string]interface{}) + err = yaml.Unmarshal(b, &values) + if err != nil { + return nil, fmt.Errorf("unmarshaling values from '%s' failed: %w", p, err) + } + mergedValues = transform.MergeMaps(mergedValues, values) + } + return mergedValues, nil +} + +// copyFileToPath attempts to copy in to out. It returns an error if out already exists. +func copyFileToPath(in, out string) error { + o, err := os.Create(out) + if err != nil { + return fmt.Errorf("failed to create copy target: %w", err) + } + defer o.Close() + i, err := os.Open(in) + if err != nil { + return fmt.Errorf("failed to open file to copy from: %w", err) + } + defer i.Close() + if _, err := o.ReadFrom(i); err != nil { + return fmt.Errorf("failed to read from source during copy: %w", err) + } + return nil +} diff --git a/internal/helm/chart/builder_local_test.go b/internal/helm/chart/builder_local_test.go new file mode 100644 index 000000000..cff5f180f --- /dev/null +++ b/internal/helm/chart/builder_local_test.go @@ -0,0 +1,384 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package chart + +import ( + "context" + "os" + "path/filepath" + "sync" + "testing" + + . "github.com/onsi/gomega" + "github.com/otiai10/copy" + helmchart "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/chartutil" + "helm.sh/helm/v3/pkg/repo" + + "github.com/fluxcd/source-controller/internal/helm/repository" +) + +func TestLocalBuilder_Build(t *testing.T) { + g := NewWithT(t) + + // Prepare chart repositories to be used for charts with remote dependency. + chartB, err := os.ReadFile("./../testdata/charts/helmchart-0.1.0.tgz") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(chartB).ToNot(BeEmpty()) + mockRepo := func() *repository.ChartRepository { + return &repository.ChartRepository{ + Client: &mockGetter{ + Response: chartB, + }, + Index: &repo.IndexFile{ + Entries: map[string]repo.ChartVersions{ + "grafana": { + &repo.ChartVersion{ + Metadata: &helmchart.Metadata{ + Name: "grafana", + Version: "6.17.4", + }, + URLs: []string{"https://example.com/grafana.tgz"}, + }, + }, + }, + }, + RWMutex: &sync.RWMutex{}, + } + } + + tests := []struct { + name string + reference Reference + buildOpts BuildOptions + valuesFiles []helmchart.File + repositories map[string]*repository.ChartRepository + dependentChartPaths []string + wantValues chartutil.Values + wantVersion string + wantPackaged bool + wantErr string + }{ + { + name: "invalid reference", + reference: RemoteReference{}, + wantErr: "expected local chart reference", + }, + { + name: "invalid local reference - no path", + reference: LocalReference{}, + wantErr: "no path set for local chart reference", + }, + { + name: "invalid local reference - no file", + reference: LocalReference{Path: "/tmp/non-existent-path.xyz"}, + wantErr: "no such file or directory", + }, + { + name: "invalid version metadata", + reference: LocalReference{Path: "./../testdata/charts/helmchart"}, + buildOpts: BuildOptions{VersionMetadata: "^"}, + wantErr: "Invalid Metadata string", + }, + { + name: "with version metadata", + reference: LocalReference{Path: "./../testdata/charts/helmchart"}, + buildOpts: BuildOptions{VersionMetadata: "foo"}, + wantVersion: "0.1.0+foo", + wantPackaged: true, + }, + { + name: "already packaged chart", + reference: LocalReference{Path: "./../testdata/charts/helmchart-0.1.0.tgz"}, + wantVersion: "0.1.0", + wantPackaged: false, + }, + { + name: "default values", + reference: LocalReference{Path: "./../testdata/charts/helmchart"}, + wantValues: chartutil.Values{ + "replicaCount": float64(1), + }, + wantVersion: "0.1.0", + wantPackaged: true, + }, + { + name: "with values files", + reference: LocalReference{Path: "./../testdata/charts/helmchart"}, + buildOpts: BuildOptions{ + ValuesFiles: []string{"custom-values1.yaml", "custom-values2.yaml"}, + }, + valuesFiles: []helmchart.File{ + { + Name: "custom-values1.yaml", + Data: []byte(`replicaCount: 11 +nameOverride: "foo-name-override"`), + }, + { + Name: "custom-values2.yaml", + Data: []byte(`replicaCount: 20 +fullnameOverride: "full-foo-name-override"`), + }, + }, + wantValues: chartutil.Values{ + "replicaCount": float64(20), + "nameOverride": "foo-name-override", + "fullnameOverride": "full-foo-name-override", + }, + wantVersion: "0.1.0", + wantPackaged: true, + }, + { + name: "chart with dependencies", + reference: LocalReference{Path: "./../testdata/charts/helmchartwithdeps"}, + repositories: map[string]*repository.ChartRepository{ + "https://grafana.github.io/helm-charts/": mockRepo(), + }, + dependentChartPaths: []string{"./../testdata/charts/helmchart"}, + wantVersion: "0.1.0", + wantPackaged: true, + }, + { + name: "v1 chart", + reference: LocalReference{Path: "./../testdata/charts/helmchart-v1"}, + wantValues: chartutil.Values{ + "replicaCount": float64(1), + }, + wantVersion: "0.2.0", + wantPackaged: true, + }, + { + name: "v1 chart with dependencies", + reference: LocalReference{Path: "./../testdata/charts/helmchartwithdeps-v1"}, + repositories: map[string]*repository.ChartRepository{ + "https://grafana.github.io/helm-charts/": mockRepo(), + }, + dependentChartPaths: []string{"./../testdata/charts/helmchart-v1"}, + wantVersion: "0.3.0", + wantPackaged: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + workDir, err := os.MkdirTemp("", "local-builder-") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(workDir) + + // Only if the reference is a LocalReference, set the WorkDir. + localRef, ok := tt.reference.(LocalReference) + if ok { + localRef.WorkDir = workDir + tt.reference = localRef + } + + // Write value file in the base dir. + for _, f := range tt.valuesFiles { + vPath := filepath.Join(workDir, f.Name) + g.Expect(os.WriteFile(vPath, f.Data, 0644)).ToNot(HaveOccurred()) + } + + // Write chart dependencies in the base dir. + for _, dcp := range tt.dependentChartPaths { + // Construct the chart path relative to the testdata chart. + helmchartDir := filepath.Join(workDir, "testdata", "charts", filepath.Base(dcp)) + g.Expect(copy.Copy(dcp, helmchartDir)).ToNot(HaveOccurred()) + } + + // Target path with name similar to the workDir. + targetPath := workDir + ".tgz" + defer os.RemoveAll(targetPath) + + dm := NewDependencyManager( + WithRepositories(tt.repositories), + ) + + b := NewLocalBuilder(dm) + cb, err := b.Build(context.TODO(), tt.reference, targetPath, tt.buildOpts) + + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(cb).To(BeZero()) + return + } + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(cb.Packaged).To(Equal(tt.wantPackaged), "unexpected Build.Packaged value") + g.Expect(cb.Path).ToNot(BeEmpty(), "empty Build.Path") + + // Load the resulting chart and verify the values. + resultChart, err := loader.Load(cb.Path) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(resultChart.Metadata.Version).To(Equal(tt.wantVersion)) + + for k, v := range tt.wantValues { + g.Expect(v).To(Equal(resultChart.Values[k])) + } + }) + } +} + +func TestLocalBuilder_Build_CachedChart(t *testing.T) { + g := NewWithT(t) + + workDir, err := os.MkdirTemp("", "local-builder-") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(workDir) + + reference := LocalReference{Path: "./../testdata/charts/helmchart"} + + dm := NewDependencyManager() + b := NewLocalBuilder(dm) + + tmpDir, err := os.MkdirTemp("", "local-chart-") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpDir) + + // Build first time. + targetPath := filepath.Join(tmpDir, "chart1.tgz") + buildOpts := BuildOptions{} + cb, err := b.Build(context.TODO(), reference, targetPath, buildOpts) + g.Expect(err).ToNot(HaveOccurred()) + + // Set the result as the CachedChart for second build. + buildOpts.CachedChart = cb.Path + + targetPath2 := filepath.Join(tmpDir, "chart2.tgz") + defer os.RemoveAll(targetPath2) + cb, err = b.Build(context.TODO(), reference, targetPath2, buildOpts) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(cb.Path).To(Equal(targetPath)) + + // Rebuild with build option Force. + buildOpts.Force = true + cb, err = b.Build(context.TODO(), reference, targetPath2, buildOpts) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(cb.Path).To(Equal(targetPath2)) +} + +func Test_mergeFileValues(t *testing.T) { + tests := []struct { + name string + files []*helmchart.File + paths []string + want map[string]interface{} + wantErr string + }{ + { + name: "merges values from files", + files: []*helmchart.File{ + {Name: "a.yaml", Data: []byte("a: b")}, + {Name: "b.yaml", Data: []byte("b: c")}, + {Name: "c.yaml", Data: []byte("b: d")}, + }, + paths: []string{"a.yaml", "b.yaml", "c.yaml"}, + want: map[string]interface{}{ + "a": "b", + "b": "d", + }, + }, + { + name: "illegal traverse", + paths: []string{"../../../traversing/illegally/a/p/a/b"}, + wantErr: "no values file found at path '/traversing/illegally/a/p/a/b'", + }, + { + name: "unmarshal error", + files: []*helmchart.File{ + {Name: "invalid", Data: []byte("abcd")}, + }, + paths: []string{"invalid"}, + wantErr: "unmarshaling values from 'invalid' failed", + }, + { + name: "error on invalid path", + paths: []string{"a.yaml"}, + wantErr: "no values file found at path '/a.yaml'", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + baseDir, err := os.MkdirTemp("", "merge-file-values-*") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(baseDir) + + for _, f := range tt.files { + g.Expect(os.WriteFile(filepath.Join(baseDir, f.Name), f.Data, 0644)).To(Succeed()) + } + + got, err := mergeFileValues(baseDir, tt.paths) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(got).To(BeNil()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(Equal(tt.want)) + }) + } +} + +func Test_copyFileToPath(t *testing.T) { + tests := []struct { + name string + in string + wantErr string + }{ + { + name: "copies input file", + in: "../testdata/local-index.yaml", + }, + { + name: "invalid input file", + in: "../testdata/invalid.tgz", + wantErr: "failed to open file to copy from", + }, + { + name: "invalid input directory", + in: "../testdata/charts", + wantErr: "failed to read from source during copy", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + out := tmpFile("copy-0.1.0", ".tgz") + defer os.RemoveAll(out) + err := copyFileToPath(tt.in, out) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(out).To(BeARegularFile()) + f1, err := os.ReadFile(tt.in) + g.Expect(err).ToNot(HaveOccurred()) + f2, err := os.ReadFile(out) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(f2).To(Equal(f1)) + }) + } +} diff --git a/internal/helm/chart/builder_remote.go b/internal/helm/chart/builder_remote.go new file mode 100644 index 000000000..617e2ec5e --- /dev/null +++ b/internal/helm/chart/builder_remote.go @@ -0,0 +1,229 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package chart + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/Masterminds/semver/v3" + helmchart "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/chartutil" + "sigs.k8s.io/yaml" + + "github.com/fluxcd/pkg/runtime/transform" + + "github.com/fluxcd/source-controller/internal/fs" + "github.com/fluxcd/source-controller/internal/helm/repository" +) + +type remoteChartBuilder struct { + remote *repository.ChartRepository +} + +// NewRemoteBuilder returns a Builder capable of building a Helm +// chart with a RemoteReference in the given repository.ChartRepository. +func NewRemoteBuilder(repository *repository.ChartRepository) Builder { + return &remoteChartBuilder{ + remote: repository, + } +} + +// Build attempts to build a Helm chart with the given RemoteReference and +// BuildOptions, writing it to p. +// It returns a Build describing the produced (or from cache observed) chart +// written to p, or a BuildError. +// +// The latest version for the RemoteReference.Version is determined in the +// repository.ChartRepository, only downloading it if the version (including +// BuildOptions.VersionMetadata) differs from the current BuildOptions.CachedChart. +// BuildOptions.ValuesFiles changes are in this case not taken into account, +// and BuildOptions.Force should be used to enforce a rebuild. +// +// After downloading the chart, it is only packaged if required due to BuildOptions +// modifying the chart, otherwise the exact data as retrieved from the repository +// is written to p, after validating it to be a chart. +func (b *remoteChartBuilder) Build(_ context.Context, ref Reference, p string, opts BuildOptions) (*Build, error) { + remoteRef, ok := ref.(RemoteReference) + if !ok { + err := fmt.Errorf("expected remote chart reference") + return nil, &BuildError{Reason: ErrChartReference, Err: err} + } + + if err := ref.Validate(); err != nil { + return nil, &BuildError{Reason: ErrChartReference, Err: err} + } + + if err := b.remote.LoadFromCache(); err != nil { + err = fmt.Errorf("could not load repository index for remote chart reference: %w", err) + return nil, &BuildError{Reason: ErrChartPull, Err: err} + } + defer b.remote.Unload() + + // Get the current version for the RemoteReference + cv, err := b.remote.Get(remoteRef.Name, remoteRef.Version) + if err != nil { + err = fmt.Errorf("failed to get chart version for remote reference: %w", err) + return nil, &BuildError{Reason: ErrChartPull, Err: err} + } + + result := &Build{} + result.Name = cv.Name + result.Version = cv.Version + // Set build specific metadata if instructed + if opts.VersionMetadata != "" { + ver, err := semver.NewVersion(result.Version) + if err != nil { + err = fmt.Errorf("failed to parse version from chart metadata as SemVer: %w", err) + return nil, &BuildError{Reason: ErrChartMetadataPatch, Err: err} + } + if *ver, err = ver.SetMetadata(opts.VersionMetadata); err != nil { + err = fmt.Errorf("failed to set SemVer metadata on chart version: %w", err) + return nil, &BuildError{Reason: ErrChartMetadataPatch, Err: err} + } + result.Version = ver.String() + } + + // If all the following is true, we do not need to download and/or build the chart: + // - Chart name from cached chart matches resolved name + // - Chart version from cached chart matches calculated version + // - BuildOptions.Force is False + if opts.CachedChart != "" && !opts.Force { + if curMeta, err := LoadChartMetadataFromArchive(opts.CachedChart); err == nil { + if result.Name == curMeta.Name && result.Version == curMeta.Version { + result.Path = opts.CachedChart + result.ValuesFiles = opts.GetValuesFiles() + return result, nil + } + } + } + + // Download the package for the resolved version + res, err := b.remote.DownloadChart(cv) + if err != nil { + err = fmt.Errorf("failed to download chart for remote reference: %w", err) + return nil, &BuildError{Reason: ErrChartPull, Err: err} + } + + // Use literal chart copy from remote if no custom values files options are + // set or build option version metadata isn't set. + if len(opts.GetValuesFiles()) == 0 && opts.VersionMetadata == "" { + if err = validatePackageAndWriteToPath(res, p); err != nil { + return nil, &BuildError{Reason: ErrChartPull, Err: err} + } + result.Path = p + return result, nil + } + + // Load the chart and merge chart values + var chart *helmchart.Chart + if chart, err = loader.LoadArchive(res); err != nil { + err = fmt.Errorf("failed to load downloaded chart: %w", err) + return nil, &BuildError{Reason: ErrChartPackage, Err: err} + } + chart.Metadata.Version = result.Version + + mergedValues, err := mergeChartValues(chart, opts.ValuesFiles) + if err != nil { + err = fmt.Errorf("failed to merge chart values: %w", err) + return nil, &BuildError{Reason: ErrValuesFilesMerge, Err: err} + } + // Overwrite default values with merged values, if any + if ok, err = OverwriteChartDefaultValues(chart, mergedValues); ok || err != nil { + if err != nil { + return nil, &BuildError{Reason: ErrValuesFilesMerge, Err: err} + } + result.ValuesFiles = opts.GetValuesFiles() + } + + // Package the chart with the custom values + if err = packageToPath(chart, p); err != nil { + return nil, &BuildError{Reason: ErrChartPackage, Err: err} + } + result.Path = p + result.Packaged = true + return result, nil +} + +// mergeChartValues merges the given chart.Chart Files paths into a single "values.yaml" map. +// It returns the merge result, or an error. +func mergeChartValues(chart *helmchart.Chart, paths []string) (map[string]interface{}, error) { + mergedValues := make(map[string]interface{}) + for _, p := range paths { + cfn := filepath.Clean(p) + if cfn == chartutil.ValuesfileName { + mergedValues = transform.MergeMaps(mergedValues, chart.Values) + continue + } + var b []byte + for _, f := range chart.Files { + if f.Name == cfn { + b = f.Data + break + } + } + if b == nil { + return nil, fmt.Errorf("no values file found at path '%s'", p) + } + values := make(map[string]interface{}) + if err := yaml.Unmarshal(b, &values); err != nil { + return nil, fmt.Errorf("unmarshaling values from '%s' failed: %w", p, err) + } + mergedValues = transform.MergeMaps(mergedValues, values) + } + return mergedValues, nil +} + +// validatePackageAndWriteToPath atomically writes the packaged chart from reader +// to out while validating it by loading the chart metadata from the archive. +func validatePackageAndWriteToPath(reader io.Reader, out string) error { + tmpFile, err := os.CreateTemp("", filepath.Base(out)) + if err != nil { + return fmt.Errorf("failed to create temporary file for chart: %w", err) + } + defer os.Remove(tmpFile.Name()) + if _, err = tmpFile.ReadFrom(reader); err != nil { + _ = tmpFile.Close() + return fmt.Errorf("failed to write chart to file: %w", err) + } + if err = tmpFile.Close(); err != nil { + return err + } + if _, err = LoadChartMetadataFromArchive(tmpFile.Name()); err != nil { + return fmt.Errorf("failed to load chart metadata from written chart: %w", err) + } + if err = fs.RenameWithFallback(tmpFile.Name(), out); err != nil { + return fmt.Errorf("failed to write chart to file: %w", err) + } + return nil +} + +// pathIsDir returns a boolean indicating if the given path points to a directory. +// In case os.Stat on the given path returns an error it returns false as well. +func pathIsDir(p string) bool { + if p == "" { + return false + } + if i, err := os.Stat(p); err != nil || !i.IsDir() { + return false + } + return true +} diff --git a/internal/helm/chart/builder_remote_test.go b/internal/helm/chart/builder_remote_test.go new file mode 100644 index 000000000..56c1fd855 --- /dev/null +++ b/internal/helm/chart/builder_remote_test.go @@ -0,0 +1,383 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package chart + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + . "github.com/onsi/gomega" + helmchart "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/chartutil" + helmgetter "helm.sh/helm/v3/pkg/getter" + + "github.com/fluxcd/source-controller/internal/helm/repository" +) + +// mockIndexChartGetter returns specific response for index and chart queries. +type mockIndexChartGetter struct { + IndexResponse []byte + ChartResponse []byte + requestedURL string +} + +func (g *mockIndexChartGetter) Get(u string, _ ...helmgetter.Option) (*bytes.Buffer, error) { + g.requestedURL = u + r := g.ChartResponse + if strings.HasSuffix(u, "index.yaml") { + r = g.IndexResponse + } + return bytes.NewBuffer(r), nil +} + +func (g *mockIndexChartGetter) LastGet() string { + return g.requestedURL +} + +func TestRemoteBuilder_Build(t *testing.T) { + g := NewWithT(t) + + chartGrafana, err := os.ReadFile("./../testdata/charts/helmchart-0.1.0.tgz") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(chartGrafana).ToNot(BeEmpty()) + + index := []byte(` +apiVersion: v1 +entries: + grafana: + - urls: + - https://example.com/grafana.tgz + description: string + version: 6.17.4 +`) + + mockGetter := &mockIndexChartGetter{ + IndexResponse: index, + ChartResponse: chartGrafana, + } + + mockRepo := func() *repository.ChartRepository { + return &repository.ChartRepository{ + URL: "https://grafana.github.io/helm-charts/", + Client: mockGetter, + RWMutex: &sync.RWMutex{}, + } + } + + tests := []struct { + name string + reference Reference + buildOpts BuildOptions + repository *repository.ChartRepository + wantValues chartutil.Values + wantVersion string + wantPackaged bool + wantErr string + }{ + { + name: "invalid reference", + reference: LocalReference{}, + wantErr: "expected remote chart reference", + }, + { + name: "invalid reference - no name", + reference: RemoteReference{}, + wantErr: "no name set for remote chart reference", + }, + { + name: "chart not in repo", + reference: RemoteReference{Name: "foo"}, + repository: mockRepo(), + wantErr: "failed to get chart version for remote reference", + }, + { + name: "chart version not in repo", + reference: RemoteReference{Name: "grafana", Version: "1.1.1"}, + repository: mockRepo(), + wantErr: "failed to get chart version for remote reference", + }, + { + name: "invalid version metadata", + reference: RemoteReference{Name: "grafana"}, + repository: mockRepo(), + buildOpts: BuildOptions{VersionMetadata: "^"}, + wantErr: "Invalid Metadata string", + }, + { + name: "with version metadata", + reference: RemoteReference{Name: "grafana"}, + repository: mockRepo(), + buildOpts: BuildOptions{VersionMetadata: "foo"}, + wantVersion: "6.17.4+foo", + wantPackaged: true, + }, + { + name: "default values", + reference: RemoteReference{Name: "grafana"}, + repository: mockRepo(), + wantVersion: "0.1.0", + wantValues: chartutil.Values{ + "replicaCount": float64(1), + }, + }, + { + name: "merge values", + reference: RemoteReference{Name: "grafana"}, + buildOpts: BuildOptions{ + ValuesFiles: []string{"a.yaml", "b.yaml", "c.yaml"}, + }, + repository: mockRepo(), + wantVersion: "6.17.4", + wantValues: chartutil.Values{ + "a": "b", + "b": "d", + }, + wantPackaged: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + tmpDir, err := os.MkdirTemp("", "remote-chart-builder-") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpDir) + targetPath := filepath.Join(tmpDir, "chart.tgz") + + if tt.repository != nil { + _, err := tt.repository.CacheIndex() + g.Expect(err).ToNot(HaveOccurred()) + // Cleanup the cache index path. + defer os.Remove(tt.repository.CachePath) + } + + b := NewRemoteBuilder(tt.repository) + + cb, err := b.Build(context.TODO(), tt.reference, targetPath, tt.buildOpts) + + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(cb).To(BeZero()) + return + } + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(cb.Packaged).To(Equal(tt.wantPackaged), "unexpected Build.Packaged value") + g.Expect(cb.Path).ToNot(BeEmpty(), "empty Build.Path") + + // Load the resulting chart and verify the values. + resultChart, err := loader.Load(cb.Path) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(resultChart.Metadata.Version).To(Equal(tt.wantVersion)) + + for k, v := range tt.wantValues { + g.Expect(v).To(Equal(resultChart.Values[k])) + } + }) + } +} + +func TestRemoteBuilder_Build_CachedChart(t *testing.T) { + g := NewWithT(t) + + chartGrafana, err := os.ReadFile("./../testdata/charts/helmchart-0.1.0.tgz") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(chartGrafana).ToNot(BeEmpty()) + + index := []byte(` +apiVersion: v1 +entries: + helmchart: + - urls: + - https://example.com/helmchart-0.1.0.tgz + description: string + version: 0.1.0 + name: helmchart +`) + + mockGetter := &mockIndexChartGetter{ + IndexResponse: index, + ChartResponse: chartGrafana, + } + mockRepo := func() *repository.ChartRepository { + return &repository.ChartRepository{ + URL: "https://grafana.github.io/helm-charts/", + Client: mockGetter, + RWMutex: &sync.RWMutex{}, + } + } + + reference := RemoteReference{Name: "helmchart"} + repository := mockRepo() + + _, err = repository.CacheIndex() + g.Expect(err).ToNot(HaveOccurred()) + // Cleanup the cache index path. + defer os.Remove(repository.CachePath) + + b := NewRemoteBuilder(repository) + + tmpDir, err := os.MkdirTemp("", "remote-chart-") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpDir) + + // Build first time. + targetPath := filepath.Join(tmpDir, "chart1.tgz") + defer os.RemoveAll(targetPath) + buildOpts := BuildOptions{} + cb, err := b.Build(context.TODO(), reference, targetPath, buildOpts) + g.Expect(err).ToNot(HaveOccurred()) + + // Set the result as the CachedChart for second build. + buildOpts.CachedChart = cb.Path + + // Rebuild with a new path. + targetPath2 := filepath.Join(tmpDir, "chart2.tgz") + defer os.RemoveAll(targetPath2) + cb, err = b.Build(context.TODO(), reference, targetPath2, buildOpts) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(cb.Path).To(Equal(targetPath)) + + // Rebuild with build option Force. + buildOpts.Force = true + cb, err = b.Build(context.TODO(), reference, targetPath2, buildOpts) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(cb.Path).To(Equal(targetPath2)) +} + +func Test_mergeChartValues(t *testing.T) { + tests := []struct { + name string + chart *helmchart.Chart + paths []string + want map[string]interface{} + wantErr string + }{ + { + name: "merges values", + chart: &helmchart.Chart{ + Files: []*helmchart.File{ + {Name: "a.yaml", Data: []byte("a: b")}, + {Name: "b.yaml", Data: []byte("b: c")}, + {Name: "c.yaml", Data: []byte("b: d")}, + }, + }, + paths: []string{"a.yaml", "b.yaml", "c.yaml"}, + want: map[string]interface{}{ + "a": "b", + "b": "d", + }, + }, + { + name: "uses chart values", + chart: &helmchart.Chart{ + Files: []*helmchart.File{ + {Name: "c.yaml", Data: []byte("b: d")}, + }, + Values: map[string]interface{}{ + "a": "b", + }, + }, + paths: []string{chartutil.ValuesfileName, "c.yaml"}, + want: map[string]interface{}{ + "a": "b", + "b": "d", + }, + }, + { + name: "unmarshal error", + chart: &helmchart.Chart{ + Files: []*helmchart.File{ + {Name: "invalid", Data: []byte("abcd")}, + }, + }, + paths: []string{"invalid"}, + wantErr: "unmarshaling values from 'invalid' failed", + }, + { + name: "error on invalid path", + chart: &helmchart.Chart{}, + paths: []string{"a.yaml"}, + wantErr: "no values file found at path 'a.yaml'", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + got, err := mergeChartValues(tt.chart, tt.paths) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(got).To(BeNil()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(Equal(tt.want)) + }) + } +} + +func Test_validatePackageAndWriteToPath(t *testing.T) { + g := NewWithT(t) + + tmpDir, err := os.MkdirTemp("", "validate-pkg-chart-") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpDir) + + validF, err := os.Open("./../testdata/charts/helmchart-0.1.0.tgz") + g.Expect(err).ToNot(HaveOccurred()) + defer validF.Close() + + chartPath := filepath.Join(tmpDir, "chart.tgz") + defer os.Remove(chartPath) + err = validatePackageAndWriteToPath(validF, chartPath) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(chartPath).To(BeARegularFile()) + + emptyF, err := os.Open("./../testdata/charts/empty.tgz") + defer emptyF.Close() + g.Expect(err).ToNot(HaveOccurred()) + err = validatePackageAndWriteToPath(emptyF, filepath.Join(tmpDir, "out.tgz")) + g.Expect(err).To(HaveOccurred()) +} + +func Test_pathIsDir(t *testing.T) { + tests := []struct { + name string + p string + want bool + }{ + {name: "directory", p: "../testdata/", want: true}, + {name: "file", p: "../testdata/local-index.yaml", want: false}, + {name: "not found error", p: "../testdata/does-not-exist.yaml", want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + g.Expect(pathIsDir(tt.p)).To(Equal(tt.want)) + }) + } +} diff --git a/internal/helm/chart/builder_test.go b/internal/helm/chart/builder_test.go new file mode 100644 index 000000000..d797a209f --- /dev/null +++ b/internal/helm/chart/builder_test.go @@ -0,0 +1,219 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package chart + +import ( + "encoding/hex" + "math/rand" + "os" + "path/filepath" + "testing" + + . "github.com/onsi/gomega" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/chartutil" +) + +func TestLocalReference_Validate(t *testing.T) { + tests := []struct { + name string + ref LocalReference + wantErr string + }{ + { + name: "ref with path", + ref: LocalReference{Path: "/a/path"}, + }, + { + name: "ref with path and work dir", + ref: LocalReference{Path: "/a/path", WorkDir: "/with/a/workdir"}, + }, + { + name: "ref without path", + ref: LocalReference{WorkDir: "/just/a/workdir"}, + wantErr: "no path set for local chart reference", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + err := tt.ref.Validate() + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + return + } + g.Expect(err).ToNot(HaveOccurred()) + }) + } +} + +func TestRemoteReference_Validate(t *testing.T) { + tests := []struct { + name string + ref RemoteReference + wantErr string + }{ + { + name: "ref with name", + ref: RemoteReference{Name: "valid-chart-name"}, + }, + { + name: "ref with invalid name", + ref: RemoteReference{Name: "iNvAlID-ChArT-NAmE!"}, + wantErr: "invalid chart name 'iNvAlID-ChArT-NAmE!'", + }, + { + name: "ref with Artifactory specific invalid format", + ref: RemoteReference{Name: "i-shall/not"}, + wantErr: "invalid chart name 'i-shall/not'", + }, + { + name: "ref without name", + ref: RemoteReference{}, + wantErr: "no name set for remote chart reference", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + err := tt.ref.Validate() + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + return + } + g.Expect(err).ToNot(HaveOccurred()) + }) + } +} + +func TestBuildOptions_GetValuesFiles(t *testing.T) { + tests := []struct { + name string + valuesFiles []string + want []string + }{ + { + name: "Default values.yaml", + valuesFiles: []string{chartutil.ValuesfileName}, + want: nil, + }, + { + name: "Values files", + valuesFiles: []string{chartutil.ValuesfileName, "foo.yaml"}, + want: []string{chartutil.ValuesfileName, "foo.yaml"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + o := BuildOptions{ValuesFiles: tt.valuesFiles} + g.Expect(o.GetValuesFiles()).To(Equal(tt.want)) + }) + } +} + +func TestChartBuildResult_Summary(t *testing.T) { + tests := []struct { + name string + build *Build + want string + }{ + { + name: "Simple", + build: &Build{ + Name: "chart", + Version: "1.2.3-rc.1+bd6bf40", + }, + want: "Pulled 'chart' chart with version '1.2.3-rc.1+bd6bf40'.", + }, + { + name: "With values files", + build: &Build{ + Name: "chart", + Version: "arbitrary-version", + Packaged: true, + ValuesFiles: []string{"a.yaml", "b.yaml"}, + }, + want: "Packaged 'chart' chart with version 'arbitrary-version', with merged values files [a.yaml b.yaml].", + }, + { + name: "With dependencies", + build: &Build{ + Name: "chart", + Version: "arbitrary-version", + Packaged: true, + ResolvedDependencies: 5, + }, + want: "Packaged 'chart' chart with version 'arbitrary-version', resolving 5 dependencies before packaging.", + }, + { + name: "Empty build", + build: &Build{}, + want: "No chart build.", + }, + { + name: "Nil build", + build: nil, + want: "No chart build.", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + g.Expect(tt.build.Summary()).To(Equal(tt.want)) + }) + } +} + +func TestChartBuildResult_String(t *testing.T) { + g := NewWithT(t) + + var result *Build + g.Expect(result.String()).To(Equal("")) + result = &Build{} + g.Expect(result.String()).To(Equal("")) + result = &Build{Path: "/foo/"} + g.Expect(result.String()).To(Equal("/foo/")) +} + +func Test_packageToPath(t *testing.T) { + g := NewWithT(t) + + chart, err := loader.Load("../testdata/charts/helmchart-0.1.0.tgz") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(chart).ToNot(BeNil()) + + out := tmpFile("chart-0.1.0", ".tgz") + defer os.RemoveAll(out) + err = packageToPath(chart, out) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(out).To(BeARegularFile()) + _, err = loader.Load(out) + g.Expect(err).ToNot(HaveOccurred()) +} + +func tmpFile(prefix, suffix string) string { + randBytes := make([]byte, 16) + rand.Read(randBytes) + return filepath.Join(os.TempDir(), prefix+hex.EncodeToString(randBytes)+suffix) +} diff --git a/internal/helm/chart/dependency_manager.go b/internal/helm/chart/dependency_manager.go new file mode 100644 index 000000000..1a053e623 --- /dev/null +++ b/internal/helm/chart/dependency_manager.go @@ -0,0 +1,339 @@ +/* +Copyright 2020 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package chart + +import ( + "context" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/Masterminds/semver/v3" + securejoin "github.com/cyphar/filepath-securejoin" + "golang.org/x/sync/errgroup" + "golang.org/x/sync/semaphore" + helmchart "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" + + "github.com/fluxcd/source-controller/internal/helm/repository" +) + +// GetChartRepositoryCallback must return a repository.ChartRepository for the +// URL, or an error describing why it could not be returned. +type GetChartRepositoryCallback func(url string) (*repository.ChartRepository, error) + +// DependencyManager manages dependencies for a Helm chart. +type DependencyManager struct { + // repositories contains a map of repository.ChartRepository objects + // indexed by their repository.NormalizeURL. + // It is consulted as a lookup table for missing dependencies, based on + // the (repository) URL the dependency refers to. + repositories map[string]*repository.ChartRepository + + // getRepositoryCallback can be set to an on-demand GetChartRepositoryCallback + // whose returned result is cached to repositories. + getRepositoryCallback GetChartRepositoryCallback + + // concurrent is the number of concurrent chart-add operations during + // Build. Defaults to 1 (non-concurrent). + concurrent int64 + + // mu contains the lock for chart writes. + mu sync.Mutex +} + +// DependencyManagerOption configures an option on a DependencyManager. +type DependencyManagerOption interface { + applyToDependencyManager(dm *DependencyManager) +} + +type WithRepositories map[string]*repository.ChartRepository + +func (o WithRepositories) applyToDependencyManager(dm *DependencyManager) { + dm.repositories = o +} + +type WithRepositoryCallback GetChartRepositoryCallback + +func (o WithRepositoryCallback) applyToDependencyManager(dm *DependencyManager) { + dm.getRepositoryCallback = GetChartRepositoryCallback(o) +} + +type WithConcurrent int64 + +func (o WithConcurrent) applyToDependencyManager(dm *DependencyManager) { + dm.concurrent = int64(o) +} + +// NewDependencyManager returns a new DependencyManager configured with the given +// DependencyManagerOption list. +func NewDependencyManager(opts ...DependencyManagerOption) *DependencyManager { + dm := &DependencyManager{} + for _, v := range opts { + v.applyToDependencyManager(dm) + } + return dm +} + +// Clear iterates over the repositories, calling Unload and RemoveCache on all +// items. It returns a collection of (cache removal) errors. +func (dm *DependencyManager) Clear() []error { + var errs []error + for _, v := range dm.repositories { + v.Unload() + if err := v.RemoveCache(); err != nil { + errs = append(errs, err) + } + } + return errs +} + +// Build compiles a set of missing dependencies from chart.Chart, and attempts to +// resolve and build them using the information from Reference. +// It returns the number of resolved local and remote dependencies, or an error. +func (dm *DependencyManager) Build(ctx context.Context, ref Reference, chart *helmchart.Chart) (int, error) { + // Collect dependency metadata + var ( + deps = chart.Dependencies() + reqs = chart.Metadata.Dependencies + ) + // Lock file takes precedence + if lock := chart.Lock; lock != nil { + reqs = lock.Dependencies + } + + // Collect missing dependencies + missing := collectMissing(deps, reqs) + if len(missing) == 0 { + return 0, nil + } + + // Run the build for the missing dependencies + if err := dm.build(ctx, ref, chart, missing); err != nil { + return 0, err + } + return len(missing), nil +} + +// chartWithLock holds a chart.Chart with a sync.Mutex to lock for writes. +type chartWithLock struct { + *helmchart.Chart + mu sync.Mutex +} + +// build adds the given list of deps to the chart with the configured number of +// concurrent workers. If the chart.Chart references a local dependency but no +// LocalReference is given, or any dependency could not be added, an error +// is returned. The first error it encounters cancels all other workers. +func (dm *DependencyManager) build(ctx context.Context, ref Reference, c *helmchart.Chart, deps map[string]*helmchart.Dependency) error { + current := dm.concurrent + if current <= 0 { + current = 1 + } + + group, groupCtx := errgroup.WithContext(ctx) + group.Go(func() error { + sem := semaphore.NewWeighted(current) + c := &chartWithLock{Chart: c} + for name, dep := range deps { + name, dep := name, dep + if err := sem.Acquire(groupCtx, 1); err != nil { + return err + } + group.Go(func() (err error) { + defer sem.Release(1) + if isLocalDep(dep) { + localRef, ok := ref.(LocalReference) + if !ok { + err = fmt.Errorf("failed to add local dependency '%s': no local chart reference", name) + return + } + if err = dm.addLocalDependency(localRef, c, dep); err != nil { + err = fmt.Errorf("failed to add local dependency '%s': %w", name, err) + } + return + } + if err = dm.addRemoteDependency(c, dep); err != nil { + err = fmt.Errorf("failed to add remote dependency '%s': %w", name, err) + } + return + }) + } + return nil + }) + return group.Wait() +} + +// addLocalDependency attempts to resolve and add the given local chart.Dependency +// to the chart. +func (dm *DependencyManager) addLocalDependency(ref LocalReference, c *chartWithLock, dep *helmchart.Dependency) error { + sLocalChartPath, err := dm.secureLocalChartPath(ref, dep) + if err != nil { + return err + } + + if _, err := os.Stat(sLocalChartPath); err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("no chart found at '%s' (reference '%s')", sLocalChartPath, dep.Repository) + } + return err + } + + constraint, err := semver.NewConstraint(dep.Version) + if err != nil { + err = fmt.Errorf("invalid version/constraint format '%s': %w", dep.Version, err) + return err + } + + ch, err := loader.Load(sLocalChartPath) + if err != nil { + return fmt.Errorf("failed to load chart from '%s' (reference '%s'): %w", + strings.TrimPrefix(sLocalChartPath, ref.WorkDir), dep.Repository, err) + } + + ver, err := semver.NewVersion(ch.Metadata.Version) + if err != nil { + return err + } + + if !constraint.Check(ver) { + err = fmt.Errorf("can't get a valid version for constraint '%s'", dep.Version) + return err + } + + c.mu.Lock() + c.AddDependency(ch) + c.mu.Unlock() + return nil +} + +// addRemoteDependency attempts to resolve and add the given remote chart.Dependency +// to the chart. It locks the chartWithLock before the downloaded dependency is +// added to the chart. +func (dm *DependencyManager) addRemoteDependency(chart *chartWithLock, dep *helmchart.Dependency) error { + repo, err := dm.resolveRepository(dep.Repository) + if err != nil { + return err + } + + if err = repo.StrategicallyLoadIndex(); err != nil { + return fmt.Errorf("failed to load index for '%s': %w", dep.Name, err) + } + + ver, err := repo.Get(dep.Name, dep.Version) + if err != nil { + return err + } + res, err := repo.DownloadChart(ver) + if err != nil { + return fmt.Errorf("chart download of version '%s' failed: %w", ver.Version, err) + } + ch, err := loader.LoadArchive(res) + if err != nil { + return fmt.Errorf("failed to load downloaded archive of version '%s': %w", ver.Version, err) + } + + chart.mu.Lock() + chart.AddDependency(ch) + chart.mu.Unlock() + return nil +} + +// resolveRepository first attempts to resolve the url from the repositories, falling back +// to getRepositoryCallback if set. It returns the resolved Index, or an error. +func (dm *DependencyManager) resolveRepository(url string) (_ *repository.ChartRepository, err error) { + dm.mu.Lock() + defer dm.mu.Unlock() + + nUrl := repository.NormalizeURL(url) + if _, ok := dm.repositories[nUrl]; !ok { + if dm.getRepositoryCallback == nil { + err = fmt.Errorf("no chart repository for URL '%s'", nUrl) + return + } + if dm.repositories == nil { + dm.repositories = map[string]*repository.ChartRepository{} + } + if dm.repositories[nUrl], err = dm.getRepositoryCallback(nUrl); err != nil { + err = fmt.Errorf("failed to get chart repository for URL '%s': %w", nUrl, err) + return + } + } + return dm.repositories[nUrl], nil +} + +// secureLocalChartPath returns the secure absolute path of a local dependency. +// It does not allow the dependency's path to be outside the scope of +// LocalReference.WorkDir. +func (dm *DependencyManager) secureLocalChartPath(ref LocalReference, dep *helmchart.Dependency) (string, error) { + localUrl, err := url.Parse(dep.Repository) + if err != nil { + return "", fmt.Errorf("failed to parse alleged local chart reference: %w", err) + } + if localUrl.Scheme != "" && localUrl.Scheme != "file" { + return "", fmt.Errorf("'%s' is not a local chart reference", dep.Repository) + } + relPath, err := filepath.Rel(ref.WorkDir, ref.Path) + if err != nil { + relPath = ref.Path + } + return securejoin.SecureJoin(ref.WorkDir, filepath.Join(relPath, localUrl.Host, localUrl.Path)) +} + +// collectMissing returns a map with dependencies from reqs that are missing +// from current, indexed by their alias or name. All dependencies of a chart +// are present if len of returned map == 0. +func collectMissing(current []*helmchart.Chart, reqs []*helmchart.Dependency) map[string]*helmchart.Dependency { + // If the number of dependencies equals the number of requested + // dependencies, there are no missing dependencies + if len(current) == len(reqs) { + return nil + } + + // Build up a map of reqs that are not in current, indexed by their + // alias or name + var missing map[string]*helmchart.Dependency + for _, dep := range reqs { + name := dep.Name + if dep.Alias != "" { + name = dep.Alias + } + // Exclude existing dependencies + found := false + for _, existing := range current { + if existing.Name() == name { + found = true + } + } + if found { + continue + } + if missing == nil { + missing = map[string]*helmchart.Dependency{} + } + missing[name] = dep + } + return missing +} + +// isLocalDep returns true if the given chart.Dependency contains a local (file) path reference. +func isLocalDep(dep *helmchart.Dependency) bool { + return dep.Repository == "" || strings.HasPrefix(dep.Repository, "file://") +} diff --git a/internal/helm/chart/dependency_manager_test.go b/internal/helm/chart/dependency_manager_test.go new file mode 100644 index 000000000..04c0fc46e --- /dev/null +++ b/internal/helm/chart/dependency_manager_test.go @@ -0,0 +1,713 @@ +/* +Copyright 2020 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package chart + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "testing" + + . "github.com/onsi/gomega" + helmchart "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" + helmgetter "helm.sh/helm/v3/pkg/getter" + "helm.sh/helm/v3/pkg/repo" + + "github.com/fluxcd/source-controller/internal/helm/repository" +) + +// mockGetter is a simple mocking getter.Getter implementation, returning +// a byte response to any provided URL. +type mockGetter struct { + Response []byte +} + +func (g *mockGetter) Get(_ string, _ ...helmgetter.Option) (*bytes.Buffer, error) { + r := g.Response + return bytes.NewBuffer(r), nil +} + +func TestDependencyManager_Clear(t *testing.T) { + g := NewWithT(t) + + repos := map[string]*repository.ChartRepository{ + "with index": { + Index: repo.NewIndexFile(), + RWMutex: &sync.RWMutex{}, + }, + "cached cache path": { + CachePath: "/invalid/path/resets", + Cached: true, + RWMutex: &sync.RWMutex{}, + }, + } + + dm := NewDependencyManager(WithRepositories(repos)) + g.Expect(dm.Clear()).To(BeNil()) + g.Expect(dm.repositories).To(HaveLen(len(repos))) + for _, v := range repos { + g.Expect(v.Index).To(BeNil()) + g.Expect(v.CachePath).To(BeEmpty()) + g.Expect(v.Cached).To(BeFalse()) + } +} + +func TestDependencyManager_Build(t *testing.T) { + g := NewWithT(t) + + // Mock chart used as grafana chart in the test below. The cached repository + // takes care of the actual grafana related details in the chart index. + chartGrafana, err := os.ReadFile("./../testdata/charts/helmchart-0.1.0.tgz") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(chartGrafana).ToNot(BeEmpty()) + + mockRepo := func() *repository.ChartRepository { + return &repository.ChartRepository{ + Client: &mockGetter{ + Response: chartGrafana, + }, + Index: &repo.IndexFile{ + Entries: map[string]repo.ChartVersions{ + "grafana": { + &repo.ChartVersion{ + Metadata: &helmchart.Metadata{ + Name: "grafana", + Version: "6.17.4", + }, + URLs: []string{"https://example.com/grafana.tgz"}, + }, + }, + }, + }, + RWMutex: &sync.RWMutex{}, + } + } + + tests := []struct { + name string + baseDir string + path string + repositories map[string]*repository.ChartRepository + getChartRepositoryCallback GetChartRepositoryCallback + want int + wantChartFunc func(g *WithT, c *helmchart.Chart) + wantErr string + }{ + { + name: "build failure returns error", + baseDir: "./../testdata/charts", + path: "helmchartwithdeps", + wantErr: "failed to add remote dependency 'grafana': no chart repository for URL", + }, + { + name: "no dependencies returns zero", + baseDir: "./../testdata/charts", + path: "helmchart", + wantChartFunc: func(g *WithT, c *helmchart.Chart) { + g.Expect(c.Dependencies()).To(HaveLen(0)) + }, + want: 0, + }, + { + name: "no dependency returns zero - v1", + baseDir: "./../testdata/charts", + path: "helmchart-v1", + wantChartFunc: func(g *WithT, c *helmchart.Chart) { + g.Expect(c.Dependencies()).To(HaveLen(0)) + }, + want: 0, + }, + { + name: "build with dependencies using lock file", + baseDir: "./../testdata/charts", + path: "helmchartwithdeps", + repositories: map[string]*repository.ChartRepository{ + "https://grafana.github.io/helm-charts/": mockRepo(), + }, + getChartRepositoryCallback: func(url string) (*repository.ChartRepository, error) { + return &repository.ChartRepository{URL: "https://grafana.github.io/helm-charts/"}, nil + }, + wantChartFunc: func(g *WithT, c *helmchart.Chart) { + g.Expect(c.Dependencies()).To(HaveLen(2)) + g.Expect(c.Lock.Dependencies).To(HaveLen(3)) + }, + want: 2, + }, + { + name: "build with dependencies - v1", + baseDir: "./../testdata/charts", + path: "helmchartwithdeps-v1", + wantChartFunc: func(g *WithT, c *helmchart.Chart) { + g.Expect(c.Dependencies()).To(HaveLen(1)) + }, + want: 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + chart, err := loader.Load(filepath.Join(tt.baseDir, tt.path)) + g.Expect(err).ToNot(HaveOccurred()) + + dm := NewDependencyManager( + WithRepositories(tt.repositories), + WithRepositoryCallback(tt.getChartRepositoryCallback), + ) + got, err := dm.Build(context.TODO(), LocalReference{WorkDir: tt.baseDir, Path: tt.path}, chart) + + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(got).To(BeZero()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(Equal(tt.want)) + if tt.wantChartFunc != nil { + tt.wantChartFunc(g, chart) + } + }) + } +} + +func TestDependencyManager_build(t *testing.T) { + tests := []struct { + name string + deps map[string]*helmchart.Dependency + wantErr string + }{ + { + name: "error remote dependency", + deps: map[string]*helmchart.Dependency{ + "example": {Repository: "https://example.com"}, + }, + wantErr: "failed to add remote dependency", + }, + { + name: "error local dependency", + deps: map[string]*helmchart.Dependency{ + "example": {Repository: "file:///invalid"}, + }, + wantErr: "failed to add remote dependency", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + dm := NewDependencyManager() + err := dm.build(context.TODO(), LocalReference{}, &helmchart.Chart{}, tt.deps) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + }) + } +} + +func TestDependencyManager_addLocalDependency(t *testing.T) { + tests := []struct { + name string + dep *helmchart.Dependency + wantErr string + wantFunc func(g *WithT, c *helmchart.Chart) + }{ + { + name: "local dependency", + dep: &helmchart.Dependency{ + Name: chartName, + Version: chartVersion, + Repository: "file://../helmchart", + }, + wantFunc: func(g *WithT, c *helmchart.Chart) { + g.Expect(c.Dependencies()).To(HaveLen(1)) + }, + }, + { + name: "version not matching constraint", + dep: &helmchart.Dependency{ + Name: chartName, + Version: "0.2.0", + Repository: "file://../helmchart", + }, + wantErr: "can't get a valid version for constraint '0.2.0'", + }, + { + name: "invalid local reference", + dep: &helmchart.Dependency{ + Name: chartName, + Version: chartVersion, + Repository: "file://../../../absolutely/invalid", + }, + wantErr: "no chart found at '../testdata/charts/absolutely/invalid'", + }, + { + name: "invalid chart archive", + dep: &helmchart.Dependency{ + Name: chartName, + Version: chartVersion, + Repository: "file://../empty.tgz", + }, + wantErr: "failed to load chart from '/empty.tgz'", + }, + { + name: "invalid constraint", + dep: &helmchart.Dependency{ + Name: chartName, + Version: "invalid", + Repository: "file://../helmchart", + }, + wantErr: "invalid version/constraint format 'invalid'", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + dm := NewDependencyManager() + chart := &helmchart.Chart{} + err := dm.addLocalDependency(LocalReference{WorkDir: "../testdata/charts", Path: "helmchartwithdeps"}, + &chartWithLock{Chart: chart}, tt.dep) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + return + } + g.Expect(err).ToNot(HaveOccurred()) + + if tt.wantFunc != nil { + tt.wantFunc(g, chart) + } + }) + } +} + +func TestDependencyManager_addRemoteDependency(t *testing.T) { + g := NewWithT(t) + + chartB, err := os.ReadFile("../testdata/charts/helmchart-0.1.0.tgz") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(chartB).ToNot(BeEmpty()) + + tests := []struct { + name string + repositories map[string]*repository.ChartRepository + dep *helmchart.Dependency + wantFunc func(g *WithT, c *helmchart.Chart) + wantErr string + }{ + { + name: "adds remote dependency", + repositories: map[string]*repository.ChartRepository{ + "https://example.com/": { + Client: &mockGetter{ + Response: chartB, + }, + Index: &repo.IndexFile{ + Entries: map[string]repo.ChartVersions{ + chartName: { + &repo.ChartVersion{ + Metadata: &helmchart.Metadata{ + Name: chartName, + Version: chartVersion, + }, + URLs: []string{"https://example.com/foo.tgz"}, + }, + }, + }, + }, + RWMutex: &sync.RWMutex{}, + }, + }, + dep: &helmchart.Dependency{ + Name: chartName, + Repository: "https://example.com", + }, + wantFunc: func(g *WithT, c *helmchart.Chart) { + g.Expect(c.Dependencies()).To(HaveLen(1)) + }, + }, + { + name: "resolve repository error", + repositories: map[string]*repository.ChartRepository{}, + dep: &helmchart.Dependency{ + Repository: "https://example.com", + }, + wantErr: "no chart repository for URL", + }, + { + name: "strategic load error", + repositories: map[string]*repository.ChartRepository{ + "https://example.com/": { + CachePath: "/invalid/cache/path/foo", + RWMutex: &sync.RWMutex{}, + }, + }, + dep: &helmchart.Dependency{ + Repository: "https://example.com", + }, + wantErr: "failed to strategically load index", + }, + { + name: "repository get error", + repositories: map[string]*repository.ChartRepository{ + "https://example.com/": { + Index: &repo.IndexFile{}, + RWMutex: &sync.RWMutex{}, + }, + }, + dep: &helmchart.Dependency{ + Repository: "https://example.com", + }, + wantErr: "no chart name found", + }, + { + name: "repository version constraint error", + repositories: map[string]*repository.ChartRepository{ + "https://example.com/": { + Index: &repo.IndexFile{ + Entries: map[string]repo.ChartVersions{ + chartName: { + &repo.ChartVersion{ + Metadata: &helmchart.Metadata{ + Name: chartName, + Version: "0.1.0", + }, + }, + }, + }, + }, + RWMutex: &sync.RWMutex{}, + }, + }, + dep: &helmchart.Dependency{ + Name: chartName, + Version: "0.2.0", + Repository: "https://example.com", + }, + wantErr: fmt.Sprintf("no '%s' chart with version matching '0.2.0' found", chartName), + }, + { + name: "repository chart download error", + repositories: map[string]*repository.ChartRepository{ + "https://example.com/": { + Index: &repo.IndexFile{ + Entries: map[string]repo.ChartVersions{ + chartName: { + &repo.ChartVersion{ + Metadata: &helmchart.Metadata{ + Name: chartName, + Version: chartVersion, + }, + }, + }, + }, + }, + RWMutex: &sync.RWMutex{}, + }, + }, + dep: &helmchart.Dependency{ + Name: chartName, + Version: chartVersion, + Repository: "https://example.com", + }, + wantErr: "chart download of version '0.1.0' failed", + }, + { + name: "chart load error", + repositories: map[string]*repository.ChartRepository{ + "https://example.com/": { + Client: &mockGetter{}, + Index: &repo.IndexFile{ + Entries: map[string]repo.ChartVersions{ + chartName: { + &repo.ChartVersion{ + Metadata: &helmchart.Metadata{ + Name: chartName, + Version: chartVersion, + }, + URLs: []string{"https://example.com/foo.tgz"}, + }, + }, + }, + }, + RWMutex: &sync.RWMutex{}, + }, + }, + dep: &helmchart.Dependency{ + Name: chartName, + Version: chartVersion, + Repository: "https://example.com", + }, + wantErr: "failed to load downloaded archive of version '0.1.0'", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + dm := &DependencyManager{ + repositories: tt.repositories, + } + chart := &helmchart.Chart{} + err := dm.addRemoteDependency(&chartWithLock{Chart: chart}, tt.dep) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + return + } + g.Expect(err).ToNot(HaveOccurred()) + if tt.wantFunc != nil { + tt.wantFunc(g, chart) + } + }) + } +} + +func TestDependencyManager_resolveRepository(t *testing.T) { + tests := []struct { + name string + repositories map[string]*repository.ChartRepository + getChartRepositoryCallback GetChartRepositoryCallback + url string + want *repository.ChartRepository + wantRepositories map[string]*repository.ChartRepository + wantErr string + }{ + { + name: "resolves from repositories index", + url: "https://example.com", + repositories: map[string]*repository.ChartRepository{ + "https://example.com/": {URL: "https://example.com"}, + }, + want: &repository.ChartRepository{URL: "https://example.com"}, + }, + { + name: "resolves from callback", + url: "https://example.com", + getChartRepositoryCallback: func(url string) (*repository.ChartRepository, error) { + return &repository.ChartRepository{URL: "https://example.com"}, nil + }, + want: &repository.ChartRepository{URL: "https://example.com"}, + wantRepositories: map[string]*repository.ChartRepository{ + "https://example.com/": {URL: "https://example.com"}, + }, + }, + { + name: "error from callback", + url: "https://example.com", + getChartRepositoryCallback: func(url string) (*repository.ChartRepository, error) { + return nil, errors.New("a very unique error") + }, + wantErr: "a very unique error", + wantRepositories: map[string]*repository.ChartRepository{}, + }, + { + name: "error on not found", + url: "https://example.com", + wantErr: "no chart repository for URL", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + dm := &DependencyManager{ + repositories: tt.repositories, + getRepositoryCallback: tt.getChartRepositoryCallback, + } + + got, err := dm.resolveRepository(tt.url) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(got).To(BeNil()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(Equal(tt.want)) + if tt.wantRepositories != nil { + g.Expect(dm.repositories).To(Equal(tt.wantRepositories)) + } + }) + } +} + +func TestDependencyManager_secureLocalChartPath(t *testing.T) { + tests := []struct { + name string + baseDir string + path string + dep *helmchart.Dependency + want string + wantErr string + }{ + { + name: "secure local file path", + baseDir: "/tmp/workdir", + path: "/chart", + dep: &helmchart.Dependency{ + Repository: "../dep", + }, + want: "/tmp/workdir/dep", + }, + { + name: "insecure local file path", + baseDir: "/tmp/workdir", + path: "/", + dep: &helmchart.Dependency{ + Repository: "/../../dep", + }, + want: "/tmp/workdir/dep", + }, + { + name: "URL parse error", + dep: &helmchart.Dependency{ + Repository: ": //example.com", + }, + wantErr: "missing protocol scheme", + }, + { + name: "error on URL scheme other than file", + dep: &helmchart.Dependency{ + Repository: "https://example.com", + }, + wantErr: "not a local chart reference", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + dm := NewDependencyManager() + got, err := dm.secureLocalChartPath(LocalReference{WorkDir: tt.baseDir, Path: tt.path}, tt.dep) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + return + } + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).ToNot(BeEmpty()) + g.Expect(got).To(Equal(tt.want)) + }) + } +} + +func Test_collectMissing(t *testing.T) { + tests := []struct { + name string + current []*helmchart.Chart + reqs []*helmchart.Dependency + want map[string]*helmchart.Dependency + }{ + { + name: "one missing", + current: []*helmchart.Chart{}, + reqs: []*helmchart.Dependency{ + {Name: chartName}, + }, + want: map[string]*helmchart.Dependency{ + chartName: {Name: chartName}, + }, + }, + { + name: "alias missing", + current: []*helmchart.Chart{ + { + Metadata: &helmchart.Metadata{ + Name: chartName, + }, + }, + }, + reqs: []*helmchart.Dependency{ + {Name: chartName}, + {Name: chartName, Alias: chartName + "-alias"}, + }, + want: map[string]*helmchart.Dependency{ + chartName + "-alias": {Name: chartName, Alias: chartName + "-alias"}, + }, + }, + { + name: "all current", + current: []*helmchart.Chart{ + { + Metadata: &helmchart.Metadata{ + Name: chartName, + }, + }, + }, + reqs: []*helmchart.Dependency{ + {Name: chartName}, + }, + want: nil, + }, + { + name: "nil", + current: nil, + reqs: nil, + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + g.Expect(collectMissing(tt.current, tt.reqs)).To(Equal(tt.want)) + }) + }) + } +} + +func Test_isLocalDep(t *testing.T) { + tests := []struct { + name string + dep *helmchart.Dependency + want bool + }{ + { + name: "file protocol", + dep: &helmchart.Dependency{Repository: "file:///some/path"}, + want: true, + }, + { + name: "empty", + dep: &helmchart.Dependency{Repository: ""}, + want: true, + }, + { + name: "https url", + dep: &helmchart.Dependency{Repository: "https://example.com"}, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + g.Expect(isLocalDep(tt.dep)).To(Equal(tt.want)) + }) + } +} diff --git a/internal/helm/chart/errors.go b/internal/helm/chart/errors.go new file mode 100644 index 000000000..dddd2e298 --- /dev/null +++ b/internal/helm/chart/errors.go @@ -0,0 +1,70 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package chart + +import ( + "errors" + "fmt" +) + +// BuildErrorReason is the descriptive reason for a BuildError. +type BuildErrorReason string + +// Error returns the string representation of BuildErrorReason. +func (e BuildErrorReason) Error() string { + return string(e) +} + +// BuildError contains a wrapped Err and a Reason indicating why it occurred. +type BuildError struct { + Reason error + Err error +} + +// Error returns Err as a string, prefixed with the Reason to provide context. +func (e *BuildError) Error() string { + if e.Reason == nil { + return e.Err.Error() + } + return fmt.Sprintf("%s: %s", e.Reason.Error(), e.Err.Error()) +} + +// Is returns true if the Reason or Err equals target. +// It can be used to programmatically place an arbitrary Err in the +// context of the Builder: +// err := &BuildError{Reason: ErrChartPull, Err: errors.New("arbitrary transport error")} +// errors.Is(err, ErrChartPull) +func (e *BuildError) Is(target error) bool { + if e.Reason != nil && e.Reason == target { + return true + } + return errors.Is(e.Err, target) +} + +// Unwrap returns the underlying Err. +func (e *BuildError) Unwrap() error { + return e.Err +} + +var ( + ErrChartReference = BuildErrorReason("chart reference error") + ErrChartPull = BuildErrorReason("chart pull error") + ErrChartMetadataPatch = BuildErrorReason("chart metadata patch error") + ErrValuesFilesMerge = BuildErrorReason("values files merge error") + ErrDependencyBuild = BuildErrorReason("dependency build error") + ErrChartPackage = BuildErrorReason("chart package error") +) diff --git a/internal/helm/chart/errors_test.go b/internal/helm/chart/errors_test.go new file mode 100644 index 000000000..f006f3364 --- /dev/null +++ b/internal/helm/chart/errors_test.go @@ -0,0 +1,84 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package chart + +import ( + "errors" + "testing" + + . "github.com/onsi/gomega" +) + +func TestBuildErrorReason_Error(t *testing.T) { + g := NewWithT(t) + + err := BuildErrorReason("reason") + g.Expect(err.Error()).To(Equal("reason")) +} + +func TestBuildError_Error(t *testing.T) { + tests := []struct { + name string + err *BuildError + want string + }{ + { + name: "with reason", + err: &BuildError{ + Reason: BuildErrorReason("reason"), + Err: errors.New("error"), + }, + want: "reason: error", + }, + { + name: "without reason", + err: &BuildError{ + Err: errors.New("error"), + }, + want: "error", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + g.Expect(tt.err.Error()).To(Equal(tt.want)) + }) + } +} + +func TestBuildError_Is(t *testing.T) { + g := NewWithT(t) + + wrappedErr := errors.New("wrapped") + err := &BuildError{ + Reason: ErrChartPackage, + Err: wrappedErr, + } + + g.Expect(err.Is(ErrChartPackage)).To(BeTrue()) + g.Expect(err.Is(wrappedErr)).To(BeTrue()) + g.Expect(err.Is(ErrDependencyBuild)).To(BeFalse()) +} + +func TestBuildError_Unwrap(t *testing.T) { + g := NewWithT(t) + + wrap := errors.New("wrapped") + err := BuildError{Err: wrap} + g.Expect(err.Unwrap()).To(Equal(wrap)) +} diff --git a/internal/helm/chart/metadata.go b/internal/helm/chart/metadata.go new file mode 100644 index 000000000..f59a599b9 --- /dev/null +++ b/internal/helm/chart/metadata.go @@ -0,0 +1,225 @@ +/* +Copyright 2020 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package chart + +import ( + "archive/tar" + "bufio" + "bytes" + "compress/gzip" + "errors" + "fmt" + "io" + "os" + "path" + "path/filepath" + "reflect" + "strings" + + helmchart "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chartutil" + "sigs.k8s.io/yaml" + + "github.com/fluxcd/source-controller/internal/helm" +) + +// OverwriteChartDefaultValues overwrites the chart default values file with the given data. +func OverwriteChartDefaultValues(chart *helmchart.Chart, vals chartutil.Values) (bool, error) { + if vals == nil { + return false, nil + } + + var bVals bytes.Buffer + if len(vals) > 0 { + if err := vals.Encode(&bVals); err != nil { + return false, err + } + } + + // Replace current values file in Raw field + for _, f := range chart.Raw { + if f.Name == chartutil.ValuesfileName { + // Do nothing if contents are equal + if reflect.DeepEqual(f.Data, bVals.Bytes()) { + return false, nil + } + + // Replace in Files field + for _, f := range chart.Files { + if f.Name == chartutil.ValuesfileName { + f.Data = bVals.Bytes() + } + } + + f.Data = bVals.Bytes() + chart.Values = vals.AsMap() + return true, nil + } + } + + // This should never happen, helm charts must have a values.yaml file to be valid + return false, fmt.Errorf("failed to locate values file: %s", chartutil.ValuesfileName) +} + +// LoadChartMetadata attempts to load the chart.Metadata from the "Chart.yaml" file in the directory or archive at the +// given chartPath. It takes "requirements.yaml" files into account, and is therefore compatible with the +// chart.APIVersionV1 format. +func LoadChartMetadata(chartPath string) (meta *helmchart.Metadata, err error) { + i, err := os.Stat(chartPath) + if err != nil { + return nil, err + } + if i.IsDir() { + meta, err = LoadChartMetadataFromDir(chartPath) + return + } + meta, err = LoadChartMetadataFromArchive(chartPath) + return +} + +// LoadChartMetadataFromDir loads the chart.Metadata from the "Chart.yaml" file in the directory at the given path. +// It takes "requirements.yaml" files into account, and is therefore compatible with the chart.APIVersionV1 format. +func LoadChartMetadataFromDir(dir string) (*helmchart.Metadata, error) { + m := new(helmchart.Metadata) + + b, err := os.ReadFile(filepath.Join(dir, chartutil.ChartfileName)) + if err != nil { + return nil, err + } + err = yaml.Unmarshal(b, m) + if err != nil { + return nil, fmt.Errorf("cannot load '%s': %w", chartutil.ChartfileName, err) + } + if m.APIVersion == "" { + m.APIVersion = helmchart.APIVersionV1 + } + + fp := filepath.Join(dir, "requirements.yaml") + stat, err := os.Stat(fp) + if (err != nil && !errors.Is(err, os.ErrNotExist)) || stat != nil { + if err != nil { + return nil, err + } + if stat.IsDir() { + return nil, fmt.Errorf("'%s' is a directory", stat.Name()) + } + if stat.Size() > helm.MaxChartFileSize { + return nil, fmt.Errorf("size of '%s' exceeds '%d' bytes limit", stat.Name(), helm.MaxChartFileSize) + } + } + + b, err = os.ReadFile(fp) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + if len(b) > 0 { + if err = yaml.Unmarshal(b, m); err != nil { + return nil, fmt.Errorf("cannot load 'requirements.yaml': %w", err) + } + } + return m, nil +} + +// LoadChartMetadataFromArchive loads the chart.Metadata from the "Chart.yaml" file in the archive at the given path. +// It takes "requirements.yaml" files into account, and is therefore compatible with the chart.APIVersionV1 format. +func LoadChartMetadataFromArchive(archive string) (*helmchart.Metadata, error) { + stat, err := os.Stat(archive) + if err != nil || stat.IsDir() { + if err == nil { + err = fmt.Errorf("'%s' is a directory", stat.Name()) + } + return nil, err + } + if stat.Size() > helm.MaxChartSize { + return nil, fmt.Errorf("size of chart '%s' exceeds '%d' bytes limit", stat.Name(), helm.MaxChartSize) + } + + f, err := os.Open(archive) + if err != nil { + return nil, err + } + defer f.Close() + + r := bufio.NewReader(f) + zr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + tr := tar.NewReader(zr) + + var m *helmchart.Metadata + for { + hd, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hd.FileInfo().IsDir() { + // Use this instead of hd.Typeflag because we don't have to do any + // inference chasing. + continue + } + + switch hd.Typeflag { + // We don't want to process these extension header files. + case tar.TypeXGlobalHeader, tar.TypeXHeader: + continue + } + + // Archive could contain \ if generated on Windows + delimiter := "/" + if strings.ContainsRune(hd.Name, '\\') { + delimiter = "\\" + } + parts := strings.Split(hd.Name, delimiter) + + // We are only interested in files in the base directory + if len(parts) != 2 { + continue + } + + // Normalize the path to the / delimiter + n := strings.Join(parts[1:], delimiter) + n = strings.ReplaceAll(n, delimiter, "/") + n = path.Clean(n) + + switch parts[1] { + case chartutil.ChartfileName, "requirements.yaml": + b, err := io.ReadAll(tr) + if err != nil { + return nil, err + } + if m == nil { + m = new(helmchart.Metadata) + } + err = yaml.Unmarshal(b, m) + if err != nil { + return nil, fmt.Errorf("cannot load '%s': %w", parts[1], err) + } + if m.APIVersion == "" { + m.APIVersion = helmchart.APIVersionV1 + } + } + } + if m == nil { + return nil, fmt.Errorf("no '%s' found", chartutil.ChartfileName) + } + return m, nil +} diff --git a/internal/helm/chart/metadata_test.go b/internal/helm/chart/metadata_test.go new file mode 100644 index 000000000..07449100a --- /dev/null +++ b/internal/helm/chart/metadata_test.go @@ -0,0 +1,267 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package chart + +import ( + "os" + "path/filepath" + "testing" + + . "github.com/onsi/gomega" + "github.com/otiai10/copy" + helmchart "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chartutil" + + "github.com/fluxcd/source-controller/internal/helm" +) + +var ( + // helmPackageFile contains the path to a Helm package in the v2 format + // without any dependencies + helmPackageFile = "../testdata/charts/helmchart-0.1.0.tgz" + chartName = "helmchart" + chartVersion = "0.1.0" + + // helmPackageV1File contains the path to a Helm package in the v1 format, + // including dependencies in a requirements.yaml file which should be + // loaded + helmPackageV1File = "../testdata/charts/helmchartwithdeps-v1-0.3.0.tgz" + chartNameV1 = "helmchartwithdeps-v1" + chartVersionV1 = "0.3.0" + + originalValuesFixture = []byte(`override: original +`) + chartFilesFixture = []*helmchart.File{ + { + Name: "values.yaml", + Data: originalValuesFixture, + }, + } + chartFixture = helmchart.Chart{ + Metadata: &helmchart.Metadata{ + Name: "test", + Version: "0.1.0", + }, + Raw: chartFilesFixture, + Files: chartFilesFixture, + } +) + +func TestOverwriteChartDefaultValues(t *testing.T) { + invalidChartFixture := chartFixture + invalidChartFixture.Raw = []*helmchart.File{} + invalidChartFixture.Files = []*helmchart.File{} + + testCases := []struct { + desc string + chart helmchart.Chart + data []byte + ok bool + expectErr bool + }{ + { + desc: "invalid chart", + chart: invalidChartFixture, + data: originalValuesFixture, + expectErr: true, + }, + { + desc: "identical override", + chart: chartFixture, + data: originalValuesFixture, + }, + { + desc: "valid override", + chart: chartFixture, + ok: true, + data: []byte(`override: test +`), + }, + { + desc: "empty override", + chart: chartFixture, + ok: true, + data: []byte(``), + }, + } + for _, tt := range testCases { + t.Run(tt.desc, func(t *testing.T) { + g := NewWithT(t) + + fixture := tt.chart + vals, err := chartutil.ReadValues(tt.data) + g.Expect(err).ToNot(HaveOccurred()) + ok, err := OverwriteChartDefaultValues(&fixture, vals) + g.Expect(ok).To(Equal(tt.ok)) + + if tt.expectErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(ok).To(Equal(tt.ok)) + return + } + + if tt.ok { + for _, f := range fixture.Raw { + if f.Name == chartutil.ValuesfileName { + g.Expect(f.Data).To(Equal(tt.data)) + } + } + for _, f := range fixture.Files { + if f.Name == chartutil.ValuesfileName { + g.Expect(f.Data).To(Equal(tt.data)) + } + } + } + }) + } +} + +func TestLoadChartMetadataFromDir(t *testing.T) { + g := NewWithT(t) + + // Create a chart file that exceeds the max chart file size. + tmpDir, err := os.MkdirTemp("", "load-chart-") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpDir) + copy.Copy("../testdata/charts/helmchart", tmpDir) + bigRequirementsFile := filepath.Join(tmpDir, "requirements.yaml") + data := make([]byte, helm.MaxChartFileSize+10) + g.Expect(os.WriteFile(bigRequirementsFile, data, 0644)).ToNot(HaveOccurred()) + + tests := []struct { + name string + dir string + wantName string + wantVersion string + wantDependencyCount int + wantErr string + }{ + { + name: "Loads from dir", + dir: "../testdata/charts/helmchart", + wantName: "helmchart", + wantVersion: "0.1.0", + }, + { + name: "Loads from v1 dir including requirements.yaml", + dir: "../testdata/charts/helmchartwithdeps-v1", + wantName: chartNameV1, + wantVersion: chartVersionV1, + wantDependencyCount: 1, + }, + { + name: "Error if no Chart.yaml", + dir: "../testdata/charts/", + wantErr: "../testdata/charts/Chart.yaml: no such file or directory", + }, + { + name: "Error if file size exceeds max size", + dir: tmpDir, + wantErr: "size of 'requirements.yaml' exceeds", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + got, err := LoadChartMetadataFromDir(tt.dir) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(got).To(BeNil()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).ToNot(BeNil()) + g.Expect(got.Validate()).To(Succeed()) + g.Expect(got.Name).To(Equal(tt.wantName)) + g.Expect(got.Version).To(Equal(tt.wantVersion)) + g.Expect(got.Dependencies).To(HaveLen(tt.wantDependencyCount)) + }) + } +} + +func TestLoadChartMetadataFromArchive(t *testing.T) { + g := NewWithT(t) + + // Create a chart archive that exceeds the max chart size. + tmpDir, err := os.MkdirTemp("", "load-chart-") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpDir) + bigArchiveFile := filepath.Join(tmpDir, "chart.tgz") + data := make([]byte, helm.MaxChartSize+10) + g.Expect(os.WriteFile(bigArchiveFile, data, 0644)).ToNot(HaveOccurred()) + + tests := []struct { + name string + archive string + wantName string + wantVersion string + wantDependencyCount int + wantErr string + }{ + { + name: "Loads from archive", + archive: helmPackageFile, + wantName: chartName, + wantVersion: chartVersion, + }, + { + name: "Loads from v1 archive including requirements.yaml", + archive: helmPackageV1File, + wantName: chartNameV1, + wantVersion: chartVersionV1, + wantDependencyCount: 1, + }, + { + name: "Error on not found", + archive: "../testdata/invalid.tgz", + wantErr: "no such file or directory", + }, + { + name: "Error if no Chart.yaml", + archive: "../testdata/charts/empty.tgz", + wantErr: "no 'Chart.yaml' found", + }, + { + name: "Error if archive size exceeds max size", + archive: bigArchiveFile, + wantErr: "size of chart 'chart.tgz' exceeds", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + got, err := LoadChartMetadataFromArchive(tt.archive) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(got).To(BeNil()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).ToNot(BeNil()) + g.Expect(got.Validate()).To(Succeed()) + g.Expect(got.Name).To(Equal(tt.wantName)) + g.Expect(got.Version).To(Equal(tt.wantVersion)) + g.Expect(got.Dependencies).To(HaveLen(tt.wantDependencyCount)) + }) + } +} diff --git a/internal/helm/chart_test.go b/internal/helm/chart_test.go deleted file mode 100644 index c0b3e8c58..000000000 --- a/internal/helm/chart_test.go +++ /dev/null @@ -1,113 +0,0 @@ -/* -Copyright 2020 The Flux authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package helm - -import ( - "reflect" - "testing" - - helmchart "helm.sh/helm/v3/pkg/chart" - "helm.sh/helm/v3/pkg/chartutil" -) - -var ( - originalValuesFixture []byte = []byte("override: original") - chartFilesFixture []*helmchart.File = []*helmchart.File{ - { - Name: "values.yaml", - Data: originalValuesFixture, - }, - } - chartFixture helmchart.Chart = helmchart.Chart{ - Metadata: &helmchart.Metadata{ - Name: "test", - Version: "0.1.0", - }, - Raw: chartFilesFixture, - Files: chartFilesFixture, - } -) - -func TestOverwriteChartDefaultValues(t *testing.T) { - invalidChartFixture := chartFixture - invalidChartFixture.Raw = []*helmchart.File{} - invalidChartFixture.Files = []*helmchart.File{} - - testCases := []struct { - desc string - chart helmchart.Chart - data []byte - ok bool - expectErr bool - }{ - { - desc: "invalid chart", - chart: invalidChartFixture, - data: originalValuesFixture, - expectErr: true, - }, - { - desc: "identical override", - chart: chartFixture, - data: originalValuesFixture, - }, - { - desc: "valid override", - chart: chartFixture, - ok: true, - data: []byte("override: test"), - }, - { - desc: "empty override", - chart: chartFixture, - ok: true, - data: []byte(""), - }, - { - desc: "invalid", - chart: chartFixture, - data: []byte("!fail:"), - expectErr: true, - }, - } - for _, tt := range testCases { - t.Run(tt.desc, func(t *testing.T) { - fixture := tt.chart - ok, err := OverwriteChartDefaultValues(&fixture, tt.data) - if ok != tt.ok { - t.Fatalf("should return %v, returned %v", tt.ok, ok) - } - if err != nil && !tt.expectErr { - t.Fatalf("returned unexpected error: %v", err) - } - if err == nil && tt.expectErr { - t.Fatal("expected error") - } - - for _, f := range fixture.Raw { - if f.Name == chartutil.ValuesfileName && reflect.DeepEqual(f.Data, originalValuesFixture) && tt.ok { - t.Error("should override values.yaml in Raw field") - } - } - for _, f := range fixture.Files { - if f.Name == chartutil.ValuesfileName && reflect.DeepEqual(f.Data, originalValuesFixture) && tt.ok { - t.Error("should override values.yaml in Files field") - } - } - }) - } -} diff --git a/internal/helm/dependency_manager.go b/internal/helm/dependency_manager.go deleted file mode 100644 index 83b42d4d7..000000000 --- a/internal/helm/dependency_manager.go +++ /dev/null @@ -1,173 +0,0 @@ -/* -Copyright 2020 The Flux authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package helm - -import ( - "context" - "fmt" - "net/url" - "os" - "path/filepath" - "strings" - "sync" - - "github.com/Masterminds/semver/v3" - securejoin "github.com/cyphar/filepath-securejoin" - "golang.org/x/sync/errgroup" - helmchart "helm.sh/helm/v3/pkg/chart" - "helm.sh/helm/v3/pkg/chart/loader" -) - -// DependencyWithRepository is a container for a Helm chart dependency -// and its respective repository. -type DependencyWithRepository struct { - // Dependency holds the reference to a chart.Chart dependency. - Dependency *helmchart.Dependency - // Repository is the ChartRepository the dependency should be - // available at and can be downloaded from. If there is none, - // a local ('file://') dependency is assumed. - Repository *ChartRepository -} - -// DependencyManager manages dependencies for a Helm chart. -type DependencyManager struct { - // WorkingDir is the chroot path for dependency manager operations, - // Dependencies that hold a local (relative) path reference are not - // allowed to traverse outside this directory. - WorkingDir string - // ChartPath is the path of the Chart relative to the WorkingDir, - // the combination of the WorkingDir and ChartPath is used to - // determine the absolute path of a local dependency. - ChartPath string - // Chart holds the loaded chart.Chart from the ChartPath. - Chart *helmchart.Chart - // Dependencies contains a list of dependencies, and the respective - // repository the dependency can be found at. - Dependencies []*DependencyWithRepository - - mu sync.Mutex -} - -// Build compiles and builds the dependencies of the Chart. -func (dm *DependencyManager) Build(ctx context.Context) error { - if len(dm.Dependencies) == 0 { - return nil - } - - errs, ctx := errgroup.WithContext(ctx) - for _, i := range dm.Dependencies { - item := i - errs.Go(func() error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - - var err error - switch item.Repository { - case nil: - err = dm.addLocalDependency(item) - default: - err = dm.addRemoteDependency(item) - } - return err - }) - } - - return errs.Wait() -} - -func (dm *DependencyManager) addLocalDependency(dpr *DependencyWithRepository) error { - sLocalChartPath, err := dm.secureLocalChartPath(dpr) - if err != nil { - return err - } - - if _, err := os.Stat(sLocalChartPath); err != nil { - if os.IsNotExist(err) { - return fmt.Errorf("no chart found at '%s' (reference '%s') for dependency '%s'", - strings.TrimPrefix(sLocalChartPath, dm.WorkingDir), dpr.Dependency.Repository, dpr.Dependency.Name) - } - return err - } - - ch, err := loader.Load(sLocalChartPath) - if err != nil { - return err - } - - constraint, err := semver.NewConstraint(dpr.Dependency.Version) - if err != nil { - err := fmt.Errorf("dependency '%s' has an invalid version/constraint format: %w", dpr.Dependency.Name, err) - return err - } - - v, err := semver.NewVersion(ch.Metadata.Version) - if err != nil { - return err - } - - if !constraint.Check(v) { - err = fmt.Errorf("can't get a valid version for dependency '%s'", dpr.Dependency.Name) - return err - } - - dm.mu.Lock() - dm.Chart.AddDependency(ch) - dm.mu.Unlock() - - return nil -} - -func (dm *DependencyManager) addRemoteDependency(dpr *DependencyWithRepository) error { - if dpr.Repository == nil { - return fmt.Errorf("no ChartRepository given for '%s' dependency", dpr.Dependency.Name) - } - - chartVer, err := dpr.Repository.Get(dpr.Dependency.Name, dpr.Dependency.Version) - if err != nil { - return err - } - - res, err := dpr.Repository.DownloadChart(chartVer) - if err != nil { - return err - } - - ch, err := loader.LoadArchive(res) - if err != nil { - return err - } - - dm.mu.Lock() - dm.Chart.AddDependency(ch) - dm.mu.Unlock() - - return nil -} - -func (dm *DependencyManager) secureLocalChartPath(dep *DependencyWithRepository) (string, error) { - localUrl, err := url.Parse(dep.Dependency.Repository) - if err != nil { - return "", fmt.Errorf("failed to parse alleged local chart reference: %w", err) - } - if localUrl.Scheme != "" && localUrl.Scheme != "file" { - return "", fmt.Errorf("'%s' is not a local chart reference", dep.Dependency.Repository) - } - return securejoin.SecureJoin(dm.WorkingDir, filepath.Join(dm.ChartPath, localUrl.Host, localUrl.Path)) -} diff --git a/internal/helm/dependency_manager_test.go b/internal/helm/dependency_manager_test.go deleted file mode 100644 index 5a5def3c2..000000000 --- a/internal/helm/dependency_manager_test.go +++ /dev/null @@ -1,217 +0,0 @@ -/* -Copyright 2020 The Flux authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package helm - -import ( - "context" - "fmt" - "os" - "strings" - "testing" - - helmchart "helm.sh/helm/v3/pkg/chart" - "helm.sh/helm/v3/pkg/repo" -) - -var ( - helmPackageFile = "testdata/charts/helmchart-0.1.0.tgz" - - chartName = "helmchart" - chartVersion = "0.1.0" - chartLocalRepository = "file://../helmchart" - remoteDepFixture = helmchart.Dependency{ - Name: chartName, - Version: chartVersion, - Repository: "https://example.com/charts", - } -) - -func TestBuild_WithEmptyDependencies(t *testing.T) { - dm := DependencyManager{ - Dependencies: nil, - } - if err := dm.Build(context.TODO()); err != nil { - t.Errorf("Build() should return nil") - } -} - -func TestBuild_WithLocalChart(t *testing.T) { - tests := []struct { - name string - dep helmchart.Dependency - wantErr bool - errMsg string - }{ - { - name: "valid path", - dep: helmchart.Dependency{ - Name: chartName, - Version: chartVersion, - Repository: chartLocalRepository, - }, - }, - { - name: "valid path", - dep: helmchart.Dependency{ - Name: chartName, - Alias: "aliased", - Version: chartVersion, - Repository: chartLocalRepository, - }, - }, - { - name: "allowed traversing path", - dep: helmchart.Dependency{ - Name: chartName, - Alias: "aliased", - Version: chartVersion, - Repository: "file://../../../testdata/charts/helmchartwithdeps/../helmchart", - }, - }, - { - name: "invalid path", - dep: helmchart.Dependency{ - Name: chartName, - Version: chartVersion, - Repository: "file://../invalid", - }, - wantErr: true, - errMsg: "no chart found at", - }, - { - name: "illegal traversing path", - dep: helmchart.Dependency{ - Name: chartName, - Version: chartVersion, - Repository: "file://../../../../../controllers/testdata/charts/helmchart", - }, - wantErr: true, - errMsg: "no chart found at", - }, - { - name: "invalid version constraint format", - dep: helmchart.Dependency{ - Name: chartName, - Version: "!2.0", - Repository: chartLocalRepository, - }, - wantErr: true, - errMsg: "has an invalid version/constraint format", - }, - { - name: "invalid version", - dep: helmchart.Dependency{ - Name: chartName, - Version: chartVersion, - Repository: chartLocalRepository, - }, - wantErr: true, - errMsg: "can't get a valid version for dependency", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := chartFixture - dm := DependencyManager{ - WorkingDir: "./", - ChartPath: "testdata/charts/helmchart", - Chart: &c, - Dependencies: []*DependencyWithRepository{ - { - Dependency: &tt.dep, - Repository: nil, - }, - }, - } - - err := dm.Build(context.TODO()) - deps := dm.Chart.Dependencies() - - if (err != nil) && tt.wantErr { - if !strings.Contains(err.Error(), tt.errMsg) { - t.Errorf("Build() expected to return error: %s, got: %s", tt.errMsg, err) - } - if len(deps) > 0 { - t.Fatalf("chart expected to have no dependencies registered") - } - return - } else if err != nil { - t.Errorf("Build() not expected to return an error: %s", err) - return - } - - if len(deps) == 0 { - t.Fatalf("chart expected to have at least one dependency registered") - } - if deps[0].Metadata.Name != chartName { - t.Errorf("chart dependency has incorrect name, expected: %s, got: %s", chartName, deps[0].Metadata.Name) - } - if deps[0].Metadata.Version != chartVersion { - t.Errorf("chart dependency has incorrect version, expected: %s, got: %s", chartVersion, deps[0].Metadata.Version) - } - }) - } -} - -func TestBuild_WithRemoteChart(t *testing.T) { - chart := chartFixture - b, err := os.ReadFile(helmPackageFile) - if err != nil { - t.Fatal(err) - } - i := repo.NewIndexFile() - i.Add(&helmchart.Metadata{Name: chartName, Version: chartVersion}, fmt.Sprintf("%s-%s.tgz", chartName, chartVersion), "http://example.com/charts", "sha256:1234567890") - mg := mockGetter{response: b} - cr := &ChartRepository{ - URL: remoteDepFixture.Repository, - Index: i, - Client: &mg, - } - dm := DependencyManager{ - Chart: &chart, - Dependencies: []*DependencyWithRepository{ - { - Dependency: &remoteDepFixture, - Repository: cr, - }, - }, - } - - if err := dm.Build(context.TODO()); err != nil { - t.Errorf("Build() expected to not return error: %s", err) - } - - deps := dm.Chart.Dependencies() - if len(deps) != 1 { - t.Fatalf("chart expected to have one dependency registered") - } - if deps[0].Metadata.Name != chartName { - t.Errorf("chart dependency has incorrect name, expected: %s, got: %s", chartName, deps[0].Metadata.Name) - } - if deps[0].Metadata.Version != chartVersion { - t.Errorf("chart dependency has incorrect version, expected: %s, got: %s", chartVersion, deps[0].Metadata.Version) - } - - // When repo is not set - dm.Dependencies[0].Repository = nil - if err := dm.Build(context.TODO()); err == nil { - t.Errorf("Build() expected to return error") - } else if !strings.Contains(err.Error(), "is not a local chart reference") { - t.Errorf("Build() expected to return different error, got: %s", err) - } -} diff --git a/internal/helm/getter.go b/internal/helm/getter/getter.go similarity index 58% rename from internal/helm/getter.go rename to internal/helm/getter/getter.go index b0f07e96b..583bac5f7 100644 --- a/internal/helm/getter.go +++ b/internal/helm/getter/getter.go @@ -14,36 +14,35 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helm +package getter import ( "fmt" "os" - "path/filepath" "helm.sh/helm/v3/pkg/getter" corev1 "k8s.io/api/core/v1" ) // ClientOptionsFromSecret constructs a getter.Option slice for the given secret. -// It returns the slice, and a callback to remove temporary files. -func ClientOptionsFromSecret(secret corev1.Secret) ([]getter.Option, func(), error) { +// It returns the slice, or an error. +func ClientOptionsFromSecret(dir string, secret corev1.Secret) ([]getter.Option, error) { var opts []getter.Option basicAuth, err := BasicAuthFromSecret(secret) if err != nil { - return opts, nil, err + return opts, err } if basicAuth != nil { opts = append(opts, basicAuth) } - tlsClientConfig, cleanup, err := TLSClientConfigFromSecret(secret) + tlsClientConfig, err := TLSClientConfigFromSecret(dir, secret) if err != nil { - return opts, nil, err + return opts, err } if tlsClientConfig != nil { opts = append(opts, tlsClientConfig) } - return opts, cleanup, nil + return opts, nil } // BasicAuthFromSecret attempts to construct a basic auth getter.Option for the @@ -63,50 +62,65 @@ func BasicAuthFromSecret(secret corev1.Secret) (getter.Option, error) { } // TLSClientConfigFromSecret attempts to construct a TLS client config -// getter.Option for the given v1.Secret. It returns the getter.Option and a -// callback to remove the temporary TLS files. +// getter.Option for the given v1.Secret, placing the required TLS config +// related files in the given directory. It returns the getter.Option, or +// an error. // // Secrets with no certFile, keyFile, AND caFile are ignored, if only a // certBytes OR keyBytes is defined it returns an error. -func TLSClientConfigFromSecret(secret corev1.Secret) (getter.Option, func(), error) { +func TLSClientConfigFromSecret(dir string, secret corev1.Secret) (getter.Option, error) { certBytes, keyBytes, caBytes := secret.Data["certFile"], secret.Data["keyFile"], secret.Data["caFile"] switch { case len(certBytes)+len(keyBytes)+len(caBytes) == 0: - return nil, func() {}, nil + return nil, nil case (len(certBytes) > 0 && len(keyBytes) == 0) || (len(keyBytes) > 0 && len(certBytes) == 0): - return nil, nil, fmt.Errorf("invalid '%s' secret data: fields 'certFile' and 'keyFile' require each other's presence", + return nil, fmt.Errorf("invalid '%s' secret data: fields 'certFile' and 'keyFile' require each other's presence", secret.Name) } - // create tmp dir for TLS files - tmp, err := os.MkdirTemp("", "helm-tls-"+secret.Name) - if err != nil { - return nil, nil, err - } - cleanup := func() { os.RemoveAll(tmp) } - - var certFile, keyFile, caFile string - + var certPath, keyPath, caPath string if len(certBytes) > 0 && len(keyBytes) > 0 { - certFile = filepath.Join(tmp, "cert.crt") - if err := os.WriteFile(certFile, certBytes, 0644); err != nil { - cleanup() - return nil, nil, err + certFile, err := os.CreateTemp(dir, "cert-*.crt") + if err != nil { + return nil, err + } + if _, err = certFile.Write(certBytes); err != nil { + _ = certFile.Close() + return nil, err } - keyFile = filepath.Join(tmp, "key.crt") - if err := os.WriteFile(keyFile, keyBytes, 0644); err != nil { - cleanup() - return nil, nil, err + if err = certFile.Close(); err != nil { + return nil, err } + certPath = certFile.Name() + + keyFile, err := os.CreateTemp(dir, "key-*.crt") + if err != nil { + return nil, err + } + if _, err = keyFile.Write(keyBytes); err != nil { + _ = keyFile.Close() + return nil, err + } + if err = keyFile.Close(); err != nil { + return nil, err + } + keyPath = keyFile.Name() } if len(caBytes) > 0 { - caFile = filepath.Join(tmp, "ca.pem") - if err := os.WriteFile(caFile, caBytes, 0644); err != nil { - cleanup() - return nil, nil, err + caFile, err := os.CreateTemp(dir, "ca-*.pem") + if err != nil { + return nil, err + } + if _, err = caFile.Write(caBytes); err != nil { + _ = caFile.Close() + return nil, err + } + if err = caFile.Close(); err != nil { + return nil, err } + caPath = caFile.Name() } - return getter.WithTLSClientConfig(certFile, keyFile, caFile), cleanup, nil + return getter.WithTLSClientConfig(certPath, keyPath, caPath), nil } diff --git a/internal/helm/getter_test.go b/internal/helm/getter/getter_test.go similarity index 90% rename from internal/helm/getter_test.go rename to internal/helm/getter/getter_test.go index bd4e1058c..6437e5b35 100644 --- a/internal/helm/getter_test.go +++ b/internal/helm/getter/getter_test.go @@ -14,9 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helm +package getter import ( + "os" "testing" corev1 "k8s.io/api/core/v1" @@ -56,10 +57,14 @@ func TestClientOptionsFromSecret(t *testing.T) { secret.Data[k] = v } } - got, cleanup, err := ClientOptionsFromSecret(secret) - if cleanup != nil { - defer cleanup() + + tmpDir, err := os.MkdirTemp("", "client-opts-secret-") + if err != nil { + t.Fatal(err) } + defer os.RemoveAll(tmpDir) + + got, err := ClientOptionsFromSecret(tmpDir, secret) if err != nil { t.Errorf("ClientOptionsFromSecret() error = %v", err) return @@ -123,10 +128,14 @@ func TestTLSClientConfigFromSecret(t *testing.T) { if tt.modify != nil { tt.modify(secret) } - got, cleanup, err := TLSClientConfigFromSecret(*secret) - if cleanup != nil { - defer cleanup() + + tmpDir, err := os.MkdirTemp("", "client-opts-secret-") + if err != nil { + t.Fatal(err) } + defer os.RemoveAll(tmpDir) + + got, err := TLSClientConfigFromSecret(tmpDir, *secret) if (err != nil) != tt.wantErr { t.Errorf("TLSClientConfigFromSecret() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/internal/helm/helm.go b/internal/helm/helm.go new file mode 100644 index 000000000..854a1ab7b --- /dev/null +++ b/internal/helm/helm.go @@ -0,0 +1,29 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helm + +// This list defines a set of global variables used to ensure Helm files loaded +// into memory during runtime do not exceed defined upper bound limits. +var ( + // MaxIndexSize is the max allowed file size in bytes of a ChartRepository. + MaxIndexSize int64 = 50 << 20 + // MaxChartSize is the max allowed file size in bytes of a Helm Chart. + MaxChartSize int64 = 10 << 20 + // MaxChartFileSize is the max allowed file size in bytes of any arbitrary + // file originating from a chart. + MaxChartFileSize int64 = 5 << 20 +) diff --git a/internal/helm/repository.go b/internal/helm/repository.go deleted file mode 100644 index 49728452d..000000000 --- a/internal/helm/repository.go +++ /dev/null @@ -1,218 +0,0 @@ -/* -Copyright 2020 The Flux authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package helm - -import ( - "bytes" - "fmt" - "io" - "net/url" - "path" - "sort" - "strings" - - "github.com/Masterminds/semver/v3" - "helm.sh/helm/v3/pkg/getter" - "helm.sh/helm/v3/pkg/repo" - "sigs.k8s.io/yaml" - - "github.com/fluxcd/pkg/version" -) - -// ChartRepository represents a Helm chart repository, and the configuration -// required to download the chart index, and charts from the repository. -type ChartRepository struct { - URL string - Index *repo.IndexFile - Client getter.Getter - Options []getter.Option -} - -// NewChartRepository constructs and returns a new ChartRepository with -// the ChartRepository.Client configured to the getter.Getter for the -// repository URL scheme. It returns an error on URL parsing failures, -// or if there is no getter available for the scheme. -func NewChartRepository(repositoryURL string, providers getter.Providers, opts []getter.Option) (*ChartRepository, error) { - u, err := url.Parse(repositoryURL) - if err != nil { - return nil, err - } - c, err := providers.ByScheme(u.Scheme) - if err != nil { - return nil, err - } - return &ChartRepository{ - URL: repositoryURL, - Client: c, - Options: opts, - }, nil -} - -// Get returns the repo.ChartVersion for the given name, the version is expected -// to be a semver.Constraints compatible string. If version is empty, the latest -// stable version will be returned and prerelease versions will be ignored. -func (r *ChartRepository) Get(name, ver string) (*repo.ChartVersion, error) { - cvs, ok := r.Index.Entries[name] - if !ok { - return nil, repo.ErrNoChartName - } - if len(cvs) == 0 { - return nil, repo.ErrNoChartVersion - } - - // Check for exact matches first - if len(ver) != 0 { - for _, cv := range cvs { - if ver == cv.Version { - return cv, nil - } - } - } - - // Continue to look for a (semantic) version match - verConstraint, err := semver.NewConstraint("*") - if err != nil { - return nil, err - } - latestStable := len(ver) == 0 || ver == "*" - if !latestStable { - verConstraint, err = semver.NewConstraint(ver) - if err != nil { - return nil, err - } - } - - // Filter out chart versions that doesn't satisfy constraints if any, - // parse semver and build a lookup table - var matchedVersions semver.Collection - lookup := make(map[*semver.Version]*repo.ChartVersion) - for _, cv := range cvs { - v, err := version.ParseVersion(cv.Version) - if err != nil { - continue - } - - if !verConstraint.Check(v) { - continue - } - - matchedVersions = append(matchedVersions, v) - lookup[v] = cv - } - if len(matchedVersions) == 0 { - return nil, fmt.Errorf("no chart version found for %s-%s", name, ver) - } - - // Sort versions - sort.SliceStable(matchedVersions, func(i, j int) bool { - // Reverse - return !(func() bool { - left := matchedVersions[i] - right := matchedVersions[j] - - if !left.Equal(right) { - return left.LessThan(right) - } - - // Having chart creation timestamp at our disposal, we put package with the - // same version into a chronological order. This is especially important for - // versions that differ only by build metadata, because it is not considered - // a part of the comparable version in Semver - return lookup[left].Created.Before(lookup[right].Created) - })() - }) - - latest := matchedVersions[0] - return lookup[latest], nil -} - -// DownloadChart confirms the given repo.ChartVersion has a downloadable URL, -// and then attempts to download the chart using the Client and Options of the -// ChartRepository. It returns a bytes.Buffer containing the chart data. -func (r *ChartRepository) DownloadChart(chart *repo.ChartVersion) (*bytes.Buffer, error) { - if len(chart.URLs) == 0 { - return nil, fmt.Errorf("chart %q has no downloadable URLs", chart.Name) - } - - // TODO(hidde): according to the Helm source the first item is not - // always the correct one to pick, check for updates once in awhile. - // Ref: https://github.com/helm/helm/blob/v3.3.0/pkg/downloader/chart_downloader.go#L241 - ref := chart.URLs[0] - u, err := url.Parse(ref) - if err != nil { - err = fmt.Errorf("invalid chart URL format '%s': %w", ref, err) - return nil, err - } - - // Prepend the chart repository base URL if the URL is relative - if !u.IsAbs() { - repoURL, err := url.Parse(r.URL) - if err != nil { - err = fmt.Errorf("invalid chart repository URL format '%s': %w", r.URL, err) - return nil, err - } - q := repoURL.Query() - // Trailing slash is required for ResolveReference to work - repoURL.Path = strings.TrimSuffix(repoURL.Path, "/") + "/" - u = repoURL.ResolveReference(u) - u.RawQuery = q.Encode() - } - - return r.Client.Get(u.String(), r.Options...) -} - -// LoadIndex loads the given bytes into the Index while performing -// minimal validity checks. It fails if the API version is not set -// (repo.ErrNoAPIVersion), or if the unmarshal fails. -// -// The logic is derived from and on par with: -// https://github.com/helm/helm/blob/v3.3.4/pkg/repo/index.go#L301 -func (r *ChartRepository) LoadIndex(b []byte) error { - i := &repo.IndexFile{} - if err := yaml.UnmarshalStrict(b, i); err != nil { - return err - } - if i.APIVersion == "" { - return repo.ErrNoAPIVersion - } - i.SortEntries() - r.Index = i - return nil -} - -// DownloadIndex attempts to download the chart repository index using -// the Client and set Options, and loads the index file into the Index. -// It returns an error on URL parsing and Client failures. -func (r *ChartRepository) DownloadIndex() error { - u, err := url.Parse(r.URL) - if err != nil { - return err - } - u.RawPath = path.Join(u.RawPath, "index.yaml") - u.Path = path.Join(u.Path, "index.yaml") - - res, err := r.Client.Get(u.String(), r.Options...) - if err != nil { - return err - } - b, err := io.ReadAll(res) - if err != nil { - return err - } - - return r.LoadIndex(b) -} diff --git a/internal/helm/repository/chart_repository.go b/internal/helm/repository/chart_repository.go new file mode 100644 index 000000000..8cee2e026 --- /dev/null +++ b/internal/helm/repository/chart_repository.go @@ -0,0 +1,382 @@ +/* +Copyright 2020 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repository + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/url" + "os" + "path" + "sort" + "strings" + "sync" + + "github.com/Masterminds/semver/v3" + "helm.sh/helm/v3/pkg/getter" + "helm.sh/helm/v3/pkg/repo" + "sigs.k8s.io/yaml" + + "github.com/fluxcd/pkg/version" + + "github.com/fluxcd/source-controller/internal/helm" +) + +var ErrNoChartIndex = errors.New("no chart index") + +// ChartRepository represents a Helm chart repository, and the configuration +// required to download the chart index and charts from the repository. +// All methods are thread safe unless defined otherwise. +type ChartRepository struct { + // URL the ChartRepository's index.yaml can be found at, + // without the index.yaml suffix. + URL string + // Client to use while downloading the Index or a chart from the URL. + Client getter.Getter + // Options to configure the Client with while downloading the Index + // or a chart from the URL. + Options []getter.Option + // CachePath is the path of a cached index.yaml for read-only operations. + CachePath string + // Cached indicates if the ChartRepository index.yaml has been cached + // to CachePath. + Cached bool + // Index contains a loaded chart repository index if not nil. + Index *repo.IndexFile + // Checksum contains the SHA256 checksum of the loaded chart repository + // index bytes. + Checksum string + + *sync.RWMutex +} + +// NewChartRepository constructs and returns a new ChartRepository with +// the ChartRepository.Client configured to the getter.Getter for the +// repository URL scheme. It returns an error on URL parsing failures, +// or if there is no getter available for the scheme. +func NewChartRepository(repositoryURL, cachePath string, providers getter.Providers, opts []getter.Option) (*ChartRepository, error) { + u, err := url.Parse(repositoryURL) + if err != nil { + return nil, err + } + c, err := providers.ByScheme(u.Scheme) + if err != nil { + return nil, err + } + + r := newChartRepository() + r.URL = repositoryURL + r.CachePath = cachePath + r.Client = c + r.Options = opts + return r, nil +} + +func newChartRepository() *ChartRepository { + return &ChartRepository{ + RWMutex: &sync.RWMutex{}, + } +} + +// Get returns the repo.ChartVersion for the given name, the version is expected +// to be a semver.Constraints compatible string. If version is empty, the latest +// stable version will be returned and prerelease versions will be ignored. +func (r *ChartRepository) Get(name, ver string) (*repo.ChartVersion, error) { + r.RLock() + defer r.RUnlock() + + if r.Index == nil { + return nil, ErrNoChartIndex + } + cvs, ok := r.Index.Entries[name] + if !ok { + return nil, repo.ErrNoChartName + } + if len(cvs) == 0 { + return nil, repo.ErrNoChartVersion + } + + // Check for exact matches first + if len(ver) != 0 { + for _, cv := range cvs { + if ver == cv.Version { + return cv, nil + } + } + } + + // Continue to look for a (semantic) version match + verConstraint, err := semver.NewConstraint("*") + if err != nil { + return nil, err + } + latestStable := len(ver) == 0 || ver == "*" + if !latestStable { + verConstraint, err = semver.NewConstraint(ver) + if err != nil { + return nil, err + } + } + + // Filter out chart versions that doesn't satisfy constraints if any, + // parse semver and build a lookup table + var matchedVersions semver.Collection + lookup := make(map[*semver.Version]*repo.ChartVersion) + for _, cv := range cvs { + v, err := version.ParseVersion(cv.Version) + if err != nil { + continue + } + + if !verConstraint.Check(v) { + continue + } + + matchedVersions = append(matchedVersions, v) + lookup[v] = cv + } + if len(matchedVersions) == 0 { + return nil, fmt.Errorf("no '%s' chart with version matching '%s' found", name, ver) + } + + // Sort versions + sort.SliceStable(matchedVersions, func(i, j int) bool { + // Reverse + return !(func() bool { + left := matchedVersions[i] + right := matchedVersions[j] + + if !left.Equal(right) { + return left.LessThan(right) + } + + // Having chart creation timestamp at our disposal, we put package with the + // same version into a chronological order. This is especially important for + // versions that differ only by build metadata, because it is not considered + // a part of the comparable version in Semver + return lookup[left].Created.Before(lookup[right].Created) + })() + }) + + latest := matchedVersions[0] + return lookup[latest], nil +} + +// DownloadChart confirms the given repo.ChartVersion has a downloadable URL, +// and then attempts to download the chart using the Client and Options of the +// ChartRepository. It returns a bytes.Buffer containing the chart data. +func (r *ChartRepository) DownloadChart(chart *repo.ChartVersion) (*bytes.Buffer, error) { + if len(chart.URLs) == 0 { + return nil, fmt.Errorf("chart '%s' has no downloadable URLs", chart.Name) + } + + // TODO(hidde): according to the Helm source the first item is not + // always the correct one to pick, check for updates once in awhile. + // Ref: https://github.com/helm/helm/blob/v3.3.0/pkg/downloader/chart_downloader.go#L241 + ref := chart.URLs[0] + u, err := url.Parse(ref) + if err != nil { + err = fmt.Errorf("invalid chart URL format '%s': %w", ref, err) + return nil, err + } + + // Prepend the chart repository base URL if the URL is relative + if !u.IsAbs() { + repoURL, err := url.Parse(r.URL) + if err != nil { + err = fmt.Errorf("invalid chart repository URL format '%s': %w", r.URL, err) + return nil, err + } + q := repoURL.Query() + // Trailing slash is required for ResolveReference to work + repoURL.Path = strings.TrimSuffix(repoURL.Path, "/") + "/" + u = repoURL.ResolveReference(u) + u.RawQuery = q.Encode() + } + + return r.Client.Get(u.String(), r.Options...) +} + +// LoadIndexFromBytes loads Index from the given bytes. +// It returns a repo.ErrNoAPIVersion error if the API version is not set +func (r *ChartRepository) LoadIndexFromBytes(b []byte) error { + i := &repo.IndexFile{} + if err := yaml.UnmarshalStrict(b, i); err != nil { + return err + } + if i.APIVersion == "" { + return repo.ErrNoAPIVersion + } + i.SortEntries() + + r.Lock() + r.Index = i + r.Checksum = fmt.Sprintf("%x", sha256.Sum256(b)) + r.Unlock() + return nil +} + +// LoadFromFile reads the file at the given path and loads it into Index. +func (r *ChartRepository) LoadFromFile(path string) error { + stat, err := os.Stat(path) + if err != nil || stat.IsDir() { + if err == nil { + err = fmt.Errorf("'%s' is a directory", path) + } + return err + } + if stat.Size() > helm.MaxIndexSize { + return fmt.Errorf("size of index '%s' exceeds '%d' bytes limit", stat.Name(), helm.MaxIndexSize) + } + b, err := os.ReadFile(path) + if err != nil { + return err + } + return r.LoadIndexFromBytes(b) +} + +// CacheIndex attempts to write the index from the remote into a new temporary file +// using DownloadIndex, and sets CachePath and Cached. +// It returns the SHA256 checksum of the downloaded index bytes, or an error. +// The caller is expected to handle the garbage collection of CachePath, and to +// load the Index separately using LoadFromCache if required. +func (r *ChartRepository) CacheIndex() (string, error) { + f, err := os.CreateTemp("", "chart-index-*.yaml") + if err != nil { + return "", fmt.Errorf("failed to create temp file to cache index to: %w", err) + } + + h := sha256.New() + mw := io.MultiWriter(f, h) + if err = r.DownloadIndex(mw); err != nil { + f.Close() + os.RemoveAll(f.Name()) + return "", fmt.Errorf("failed to cache index to '%s': %w", f.Name(), err) + } + if err = f.Close(); err != nil { + os.RemoveAll(f.Name()) + return "", fmt.Errorf("failed to close cached index file '%s': %w", f.Name(), err) + } + + r.Lock() + r.CachePath = f.Name() + r.Cached = true + r.Unlock() + return hex.EncodeToString(h.Sum(nil)), nil +} + +// StrategicallyLoadIndex lazy-loads the Index from CachePath using +// LoadFromCache if it does not HasIndex. +// If not HasCacheFile, a cache attempt is made using CacheIndex +// before continuing to load. +func (r *ChartRepository) StrategicallyLoadIndex() (err error) { + if r.HasIndex() { + return + } + if !r.HasCacheFile() { + if _, err = r.CacheIndex(); err != nil { + err = fmt.Errorf("failed to strategically load index: %w", err) + return + } + } + if err = r.LoadFromCache(); err != nil { + err = fmt.Errorf("failed to strategically load index: %w", err) + return + } + return +} + +// LoadFromCache attempts to load the Index from the configured CachePath. +// It returns an error if no CachePath is set, or if the load failed. +func (r *ChartRepository) LoadFromCache() error { + if cachePath := r.CachePath; cachePath != "" { + return r.LoadFromFile(cachePath) + } + return fmt.Errorf("no cache path set") +} + +// DownloadIndex attempts to download the chart repository index using +// the Client and set Options, and writes the index to the given io.Writer. +// It returns an url.Error if the URL failed to parse. +func (r *ChartRepository) DownloadIndex(w io.Writer) (err error) { + u, err := url.Parse(r.URL) + if err != nil { + return err + } + u.RawPath = path.Join(u.RawPath, "index.yaml") + u.Path = path.Join(u.Path, "index.yaml") + + var res *bytes.Buffer + res, err = r.Client.Get(u.String(), r.Options...) + if err != nil { + return err + } + if _, err = io.Copy(w, res); err != nil { + return err + } + return nil +} + +// HasIndex returns true if the Index is not nil. +func (r *ChartRepository) HasIndex() bool { + r.RLock() + defer r.RUnlock() + return r.Index != nil +} + +// HasCacheFile returns true if CachePath is not empty. +func (r *ChartRepository) HasCacheFile() bool { + r.RLock() + defer r.RUnlock() + return r.CachePath != "" +} + +// Unload can be used to signal the Go garbage collector the Index can +// be freed from memory if the ChartRepository object is expected to +// continue to exist in the stack for some time. +func (r *ChartRepository) Unload() { + if r == nil { + return + } + + r.Lock() + defer r.Unlock() + r.Index = nil +} + +// RemoveCache removes the CachePath if Cached. +func (r *ChartRepository) RemoveCache() error { + if r == nil { + return nil + } + + r.Lock() + defer r.Unlock() + + if r.Cached { + if err := os.Remove(r.CachePath); err != nil && !os.IsNotExist(err) { + return err + } + r.CachePath = "" + r.Cached = false + } + return nil +} diff --git a/internal/helm/repository/chart_repository_test.go b/internal/helm/repository/chart_repository_test.go new file mode 100644 index 000000000..c0100dd3d --- /dev/null +++ b/internal/helm/repository/chart_repository_test.go @@ -0,0 +1,616 @@ +/* +Copyright 2020 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repository + +import ( + "bytes" + "crypto/sha256" + "fmt" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + "github.com/fluxcd/source-controller/internal/helm" + . "github.com/onsi/gomega" + "helm.sh/helm/v3/pkg/chart" + helmgetter "helm.sh/helm/v3/pkg/getter" + "helm.sh/helm/v3/pkg/repo" +) + +var now = time.Now() + +const ( + testFile = "../testdata/local-index.yaml" + chartmuseumTestFile = "../testdata/chartmuseum-index.yaml" + unorderedTestFile = "../testdata/local-index-unordered.yaml" +) + +// mockGetter is a simple mocking getter.Getter implementation, returning +// a byte response to any provided URL. +type mockGetter struct { + Response []byte + LastCalledURL string +} + +func (g *mockGetter) Get(u string, _ ...helmgetter.Option) (*bytes.Buffer, error) { + r := g.Response + g.LastCalledURL = u + return bytes.NewBuffer(r), nil +} + +func TestNewChartRepository(t *testing.T) { + repositoryURL := "https://example.com" + providers := helmgetter.Providers{ + helmgetter.Provider{ + Schemes: []string{"https"}, + New: helmgetter.NewHTTPGetter, + }, + } + options := []helmgetter.Option{helmgetter.WithBasicAuth("username", "password")} + + t.Run("should construct chart repository", func(t *testing.T) { + g := NewWithT(t) + + r, err := NewChartRepository(repositoryURL, "", providers, options) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(r).ToNot(BeNil()) + g.Expect(r.URL).To(Equal(repositoryURL)) + g.Expect(r.Client).ToNot(BeNil()) + g.Expect(r.Options).To(Equal(options)) + }) + + t.Run("should error on URL parsing failure", func(t *testing.T) { + g := NewWithT(t) + r, err := NewChartRepository("https://ex ample.com", "", nil, nil) + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(BeAssignableToTypeOf(&url.Error{})) + g.Expect(r).To(BeNil()) + + }) + + t.Run("should error on unsupported scheme", func(t *testing.T) { + g := NewWithT(t) + + r, err := NewChartRepository("http://example.com", "", providers, nil) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(Equal("scheme \"http\" not supported")) + g.Expect(r).To(BeNil()) + }) +} + +func TestChartRepository_Get(t *testing.T) { + g := NewWithT(t) + + r := newChartRepository() + r.Index = repo.NewIndexFile() + charts := []struct { + name string + version string + url string + digest string + created time.Time + }{ + {name: "chart", version: "0.0.1", url: "http://example.com/charts", digest: "sha256:1234567890"}, + {name: "chart", version: "0.1.0", url: "http://example.com/charts", digest: "sha256:1234567890abc"}, + {name: "chart", version: "0.1.1", url: "http://example.com/charts", digest: "sha256:1234567890abc"}, + {name: "chart", version: "0.1.5+b.min.minute", url: "http://example.com/charts", digest: "sha256:1234567890abc", created: now.Add(-time.Minute)}, + {name: "chart", version: "0.1.5+a.min.hour", url: "http://example.com/charts", digest: "sha256:1234567890abc", created: now.Add(-time.Hour)}, + {name: "chart", version: "0.1.5+c.now", url: "http://example.com/charts", digest: "sha256:1234567890abc", created: now}, + {name: "chart", version: "0.2.0", url: "http://example.com/charts", digest: "sha256:1234567890abc"}, + {name: "chart", version: "1.0.0", url: "http://example.com/charts", digest: "sha256:1234567890abc"}, + {name: "chart", version: "1.1.0-rc.1", url: "http://example.com/charts", digest: "sha256:1234567890abc"}, + } + for _, c := range charts { + g.Expect(r.Index.MustAdd( + &chart.Metadata{Name: c.name, Version: c.version}, + fmt.Sprintf("%s-%s.tgz", c.name, c.version), c.url, c.digest), + ).To(Succeed()) + if !c.created.IsZero() { + r.Index.Entries["chart"][len(r.Index.Entries["chart"])-1].Created = c.created + } + } + r.Index.SortEntries() + + tests := []struct { + name string + chartName string + chartVersion string + wantVersion string + wantErr string + }{ + { + name: "exact match", + chartName: "chart", + chartVersion: "0.0.1", + wantVersion: "0.0.1", + }, + { + name: "stable version", + chartName: "chart", + chartVersion: "", + wantVersion: "1.0.0", + }, + { + name: "stable version (asterisk)", + chartName: "chart", + chartVersion: "*", + wantVersion: "1.0.0", + }, + { + name: "semver range", + chartName: "chart", + chartVersion: "<1.0.0", + wantVersion: "0.2.0", + }, + { + name: "unfulfilled range", + chartName: "chart", + chartVersion: ">2.0.0", + wantErr: "no 'chart' chart with version matching '>2.0.0' found", + }, + { + name: "invalid chart", + chartName: "non-existing", + wantErr: repo.ErrNoChartName.Error(), + }, + { + name: "match newest if ambiguous", + chartName: "chart", + chartVersion: "0.1.5", + wantVersion: "0.1.5+c.now", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + cv, err := r.Get(tt.chartName, tt.chartVersion) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(cv).To(BeNil()) + return + } + g.Expect(cv).ToNot(BeNil()) + g.Expect(cv.Metadata.Name).To(Equal(tt.chartName)) + g.Expect(cv.Metadata.Version).To(Equal(tt.wantVersion)) + g.Expect(err).ToNot(HaveOccurred()) + }) + } +} + +func TestChartRepository_DownloadChart(t *testing.T) { + tests := []struct { + name string + url string + chartVersion *repo.ChartVersion + wantURL string + wantErr bool + }{ + { + name: "relative URL", + url: "https://example.com", + chartVersion: &repo.ChartVersion{ + Metadata: &chart.Metadata{Name: "chart"}, + URLs: []string{"charts/foo-1.0.0.tgz"}, + }, + wantURL: "https://example.com/charts/foo-1.0.0.tgz", + }, + { + name: "no chart URL", + chartVersion: &repo.ChartVersion{Metadata: &chart.Metadata{Name: "chart"}}, + wantErr: true, + }, + { + name: "invalid chart URL", + chartVersion: &repo.ChartVersion{ + Metadata: &chart.Metadata{Name: "chart"}, + URLs: []string{"https://ex ample.com/charts/foo-1.0.0.tgz"}, + }, + wantErr: true, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + t.Parallel() + + mg := mockGetter{} + r := &ChartRepository{ + URL: tt.url, + Client: &mg, + } + res, err := r.DownloadChart(tt.chartVersion) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(res).To(BeNil()) + return + } + g.Expect(mg.LastCalledURL).To(Equal(tt.wantURL)) + g.Expect(res).ToNot(BeNil()) + g.Expect(err).ToNot(HaveOccurred()) + }) + } +} + +func TestChartRepository_DownloadIndex(t *testing.T) { + g := NewWithT(t) + + b, err := os.ReadFile(chartmuseumTestFile) + g.Expect(err).ToNot(HaveOccurred()) + + mg := mockGetter{Response: b} + r := &ChartRepository{ + URL: "https://example.com", + Client: &mg, + } + + buf := bytes.NewBuffer([]byte{}) + g.Expect(r.DownloadIndex(buf)).To(Succeed()) + g.Expect(buf.Bytes()).To(Equal(b)) + g.Expect(mg.LastCalledURL).To(Equal(r.URL + "/index.yaml")) + g.Expect(err).To(BeNil()) +} + +func TestChartRepository_LoadIndexFromBytes(t *testing.T) { + tests := []struct { + name string + b []byte + wantName string + wantVersion string + wantDigest string + wantErr string + }{ + { + name: "index", + b: []byte(` +apiVersion: v1 +entries: + nginx: + - urls: + - https://kubernetes-charts.storage.googleapis.com/nginx-0.2.0.tgz + name: nginx + description: string + version: 0.2.0 + home: https://github.com/something/else + digest: "sha256:1234567890abcdef" +`), + wantName: "nginx", + wantVersion: "0.2.0", + wantDigest: "sha256:1234567890abcdef", + }, + { + name: "index without API version", + b: []byte(`entries: + nginx: + - name: nginx`), + wantErr: "no API version specified", + }, + { + name: "index with duplicate entry", + b: []byte(`apiVersion: v1 +entries: + nginx: + - name: nginx" + nginx: + - name: nginx`), + wantErr: "key \"nginx\" already set in map", + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + t.Parallel() + + r := newChartRepository() + err := r.LoadIndexFromBytes(tt.b) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(r.Index).To(BeNil()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(r.Index).ToNot(BeNil()) + got, err := r.Index.Get(tt.wantName, tt.wantVersion) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got.Digest).To(Equal(tt.wantDigest)) + }) + } +} + +func TestChartRepository_LoadIndexFromBytes_Unordered(t *testing.T) { + b, err := os.ReadFile(unorderedTestFile) + if err != nil { + t.Fatal(err) + } + r := newChartRepository() + err = r.LoadIndexFromBytes(b) + if err != nil { + t.Fatal(err) + } + verifyLocalIndex(t, r.Index) +} + +// Index load tests are derived from https://github.com/helm/helm/blob/v3.3.4/pkg/repo/index_test.go#L108 +// to ensure parity with Helm behaviour. +func TestChartRepository_LoadIndexFromFile(t *testing.T) { + g := NewWithT(t) + + // Create an index file that exceeds the max index size. + tmpDir, err := os.MkdirTemp("", "load-index-") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpDir) + bigIndexFile := filepath.Join(tmpDir, "index.yaml") + data := make([]byte, helm.MaxIndexSize+10) + g.Expect(os.WriteFile(bigIndexFile, data, 0644)).ToNot(HaveOccurred()) + + tests := []struct { + name string + filename string + wantErr string + }{ + { + name: "regular index file", + filename: testFile, + }, + { + name: "chartmuseum index file", + filename: chartmuseumTestFile, + }, + { + name: "error if index size exceeds max size", + filename: bigIndexFile, + wantErr: "size of index 'index.yaml' exceeds", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + r := newChartRepository() + err := r.LoadFromFile(tt.filename) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + + verifyLocalIndex(t, r.Index) + }) + } +} + +func TestChartRepository_CacheIndex(t *testing.T) { + g := NewWithT(t) + + mg := mockGetter{Response: []byte("foo")} + expectSum := fmt.Sprintf("%x", sha256.Sum256(mg.Response)) + + r := newChartRepository() + r.URL = "https://example.com" + r.Client = &mg + + sum, err := r.CacheIndex() + g.Expect(err).To(Not(HaveOccurred())) + + g.Expect(r.CachePath).ToNot(BeEmpty()) + defer os.RemoveAll(r.CachePath) + g.Expect(r.CachePath).To(BeARegularFile()) + b, _ := os.ReadFile(r.CachePath) + + g.Expect(b).To(Equal(mg.Response)) + g.Expect(sum).To(BeEquivalentTo(expectSum)) +} + +func TestChartRepository_StrategicallyLoadIndex(t *testing.T) { + g := NewWithT(t) + + r := newChartRepository() + r.Index = repo.NewIndexFile() + g.Expect(r.StrategicallyLoadIndex()).To(Succeed()) + g.Expect(r.CachePath).To(BeEmpty()) + g.Expect(r.Cached).To(BeFalse()) + + r.Index = nil + r.CachePath = "/invalid/cache/index/path.yaml" + err := r.StrategicallyLoadIndex() + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("/invalid/cache/index/path.yaml: no such file or directory")) + g.Expect(r.Cached).To(BeFalse()) + + r.CachePath = "" + r.Client = &mockGetter{} + err = r.StrategicallyLoadIndex() + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("no API version specified")) + g.Expect(r.Cached).To(BeTrue()) + g.Expect(r.RemoveCache()).To(Succeed()) +} + +func TestChartRepository_LoadFromCache(t *testing.T) { + tests := []struct { + name string + cachePath string + wantErr string + }{ + { + name: "cache path", + cachePath: chartmuseumTestFile, + }, + { + name: "invalid cache path", + cachePath: "invalid", + wantErr: "stat invalid: no such file", + }, + { + name: "no cache path", + cachePath: "", + wantErr: "no cache path set", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + r := newChartRepository() + r.CachePath = tt.cachePath + err := r.LoadFromCache() + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(r.Index).To(BeNil()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + verifyLocalIndex(t, r.Index) + }) + } +} + +func TestChartRepository_HasIndex(t *testing.T) { + g := NewWithT(t) + + r := newChartRepository() + g.Expect(r.HasIndex()).To(BeFalse()) + r.Index = repo.NewIndexFile() + g.Expect(r.HasIndex()).To(BeTrue()) +} + +func TestChartRepository_HasCacheFile(t *testing.T) { + g := NewWithT(t) + + r := newChartRepository() + g.Expect(r.HasCacheFile()).To(BeFalse()) + r.CachePath = "foo" + g.Expect(r.HasCacheFile()).To(BeTrue()) +} + +func TestChartRepository_UnloadIndex(t *testing.T) { + g := NewWithT(t) + + r := newChartRepository() + g.Expect(r.HasIndex()).To(BeFalse()) + r.Index = repo.NewIndexFile() + r.Unload() + g.Expect(r.Index).To(BeNil()) +} + +func verifyLocalIndex(t *testing.T, i *repo.IndexFile) { + g := NewWithT(t) + + g.Expect(i.Entries).ToNot(BeNil()) + g.Expect(i.Entries).To(HaveLen(3), "expected 3 entries in index file") + + alpine, ok := i.Entries["alpine"] + g.Expect(ok).To(BeTrue(), "expected 'alpine' entry to exist") + g.Expect(alpine).To(HaveLen(1), "'alpine' should have 1 entry") + + nginx, ok := i.Entries["nginx"] + g.Expect(ok).To(BeTrue(), "expected 'nginx' entry to exist") + g.Expect(nginx).To(HaveLen(2), "'nginx' should have 2 entries") + + expects := []*repo.ChartVersion{ + { + Metadata: &chart.Metadata{ + Name: "alpine", + Description: "string", + Version: "1.0.0", + Keywords: []string{"linux", "alpine", "small", "sumtin"}, + Home: "https://github.com/something", + }, + URLs: []string{ + "https://kubernetes-charts.storage.googleapis.com/alpine-1.0.0.tgz", + "http://storage2.googleapis.com/kubernetes-charts/alpine-1.0.0.tgz", + }, + Digest: "sha256:1234567890abcdef", + }, + { + Metadata: &chart.Metadata{ + Name: "nginx", + Description: "string", + Version: "0.2.0", + Keywords: []string{"popular", "web server", "proxy"}, + Home: "https://github.com/something/else", + }, + URLs: []string{ + "https://kubernetes-charts.storage.googleapis.com/nginx-0.2.0.tgz", + }, + Digest: "sha256:1234567890abcdef", + }, + { + Metadata: &chart.Metadata{ + Name: "nginx", + Description: "string", + Version: "0.1.0", + Keywords: []string{"popular", "web server", "proxy"}, + Home: "https://github.com/something", + }, + URLs: []string{ + "https://kubernetes-charts.storage.googleapis.com/nginx-0.1.0.tgz", + }, + Digest: "sha256:1234567890abcdef", + }, + } + tests := []*repo.ChartVersion{alpine[0], nginx[0], nginx[1]} + + for i, tt := range tests { + expect := expects[i] + g.Expect(tt.Name).To(Equal(expect.Name)) + g.Expect(tt.Description).To(Equal(expect.Description)) + g.Expect(tt.Version).To(Equal(expect.Version)) + g.Expect(tt.Digest).To(Equal(expect.Digest)) + g.Expect(tt.Home).To(Equal(expect.Home)) + g.Expect(tt.URLs).To(ContainElements(expect.URLs)) + g.Expect(tt.Keywords).To(ContainElements(expect.Keywords)) + } +} + +func TestChartRepository_RemoveCache(t *testing.T) { + g := NewWithT(t) + + tmpFile, err := os.CreateTemp("", "remove-cache-") + g.Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tmpFile.Name()) + + r := newChartRepository() + r.CachePath = tmpFile.Name() + r.Cached = true + + g.Expect(r.RemoveCache()).To(Succeed()) + g.Expect(r.CachePath).To(BeEmpty()) + g.Expect(r.Cached).To(BeFalse()) + g.Expect(tmpFile.Name()).ToNot(BeAnExistingFile()) + + r.CachePath = tmpFile.Name() + r.Cached = true + + g.Expect(r.RemoveCache()).To(Succeed()) + g.Expect(r.CachePath).To(BeEmpty()) + g.Expect(r.Cached).To(BeFalse()) +} diff --git a/internal/helm/utils.go b/internal/helm/repository/utils.go similarity index 77% rename from internal/helm/utils.go rename to internal/helm/repository/utils.go index ff2221c61..b02b13782 100644 --- a/internal/helm/utils.go +++ b/internal/helm/repository/utils.go @@ -1,5 +1,5 @@ /* -Copyright 2020 The Flux authors +Copyright 2021 The Flux authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,12 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helm +package repository import "strings" -// NormalizeChartRepositoryURL ensures repository urls are normalized -func NormalizeChartRepositoryURL(url string) string { +// NormalizeURL normalizes a ChartRepository URL by ensuring it ends with a +// single "/". +func NormalizeURL(url string) string { if url != "" { return strings.TrimRight(url, "/") + "/" } diff --git a/internal/helm/repository/utils_test.go b/internal/helm/repository/utils_test.go new file mode 100644 index 000000000..bac683b46 --- /dev/null +++ b/internal/helm/repository/utils_test.go @@ -0,0 +1,60 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repository + +import ( + "testing" + + . "github.com/onsi/gomega" +) + +func TestNormalizeURL(t *testing.T) { + tests := []struct { + name string + url string + want string + }{ + { + name: "with slash", + url: "http://example.com/", + want: "http://example.com/", + }, + { + name: "without slash", + url: "http://example.com", + want: "http://example.com/", + }, + { + name: "double slash", + url: "http://example.com//", + want: "http://example.com/", + }, + { + name: "empty", + url: "", + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + got := NormalizeURL(tt.url) + g.Expect(got).To(Equal(tt.want)) + }) + } +} diff --git a/internal/helm/repository_test.go b/internal/helm/repository_test.go deleted file mode 100644 index c51a19d40..000000000 --- a/internal/helm/repository_test.go +++ /dev/null @@ -1,410 +0,0 @@ -/* -Copyright 2020 The Flux authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package helm - -import ( - "bytes" - "net/url" - "os" - "reflect" - "strings" - "testing" - "time" - - "helm.sh/helm/v3/pkg/chart" - "helm.sh/helm/v3/pkg/getter" - "helm.sh/helm/v3/pkg/repo" -) - -const ( - testfile = "testdata/local-index.yaml" - chartmuseumtestfile = "testdata/chartmuseum-index.yaml" - unorderedtestfile = "testdata/local-index-unordered.yaml" - indexWithDuplicates = ` -apiVersion: v1 -entries: - nginx: - - urls: - - https://kubernetes-charts.storage.googleapis.com/nginx-0.2.0.tgz - name: nginx - description: string - version: 0.2.0 - home: https://github.com/something/else - digest: "sha256:1234567890abcdef" - nginx: - - urls: - - https://kubernetes-charts.storage.googleapis.com/alpine-1.0.0.tgz - - http://storage2.googleapis.com/kubernetes-charts/alpine-1.0.0.tgz - name: alpine - description: string - version: 1.0.0 - home: https://github.com/something - digest: "sha256:1234567890abcdef" -` -) - -func TestNewChartRepository(t *testing.T) { - repositoryURL := "https://example.com" - providers := getter.Providers{ - getter.Provider{ - Schemes: []string{"https"}, - New: getter.NewHTTPGetter, - }, - } - options := []getter.Option{getter.WithBasicAuth("username", "password")} - - t.Run("should construct chart repository", func(t *testing.T) { - r, err := NewChartRepository(repositoryURL, providers, options) - if err != nil { - t.Error(err) - } - if got := r.URL; got != repositoryURL { - t.Fatalf("Expecting %q repository URL, got: %q", repositoryURL, got) - } - if r.Client == nil { - t.Fatalf("Expecting client, got nil") - } - if !reflect.DeepEqual(r.Options, options) { - t.Fatalf("Client options mismatth") - } - }) - - t.Run("should error on URL parsing failure", func(t *testing.T) { - _, err := NewChartRepository("https://ex ample.com", nil, nil) - switch err.(type) { - case *url.Error: - default: - t.Fatalf("Expecting URL error, got: %v", err) - } - }) - - t.Run("should error on unsupported scheme", func(t *testing.T) { - _, err := NewChartRepository("http://example.com", providers, nil) - if err == nil { - t.Fatalf("Expecting unsupported scheme error") - } - }) -} - -func TestChartRepository_Get(t *testing.T) { - i := repo.NewIndexFile() - i.Add(&chart.Metadata{Name: "chart", Version: "0.0.1"}, "chart-0.0.1.tgz", "http://example.com/charts", "sha256:1234567890") - i.Add(&chart.Metadata{Name: "chart", Version: "0.1.0"}, "chart-0.1.0.tgz", "http://example.com/charts", "sha256:1234567890abc") - i.Add(&chart.Metadata{Name: "chart", Version: "0.1.1"}, "chart-0.1.1.tgz", "http://example.com/charts", "sha256:1234567890abc") - i.Add(&chart.Metadata{Name: "chart", Version: "0.1.5+b.min.minute"}, "chart-0.1.5+b.min.minute.tgz", "http://example.com/charts", "sha256:1234567890abc") - i.Entries["chart"][len(i.Entries["chart"])-1].Created = time.Now().Add(-time.Minute) - i.Add(&chart.Metadata{Name: "chart", Version: "0.1.5+a.min.hour"}, "chart-0.1.5+a.min.hour.tgz", "http://example.com/charts", "sha256:1234567890abc") - i.Entries["chart"][len(i.Entries["chart"])-1].Created = time.Now().Add(-time.Hour) - i.Add(&chart.Metadata{Name: "chart", Version: "0.1.5+c.now"}, "chart-0.1.5+c.now.tgz", "http://example.com/charts", "sha256:1234567890abc") - i.Add(&chart.Metadata{Name: "chart", Version: "0.2.0"}, "chart-0.2.0.tgz", "http://example.com/charts", "sha256:1234567890abc") - i.Add(&chart.Metadata{Name: "chart", Version: "1.0.0"}, "chart-1.0.0.tgz", "http://example.com/charts", "sha256:1234567890abc") - i.Add(&chart.Metadata{Name: "chart", Version: "1.1.0-rc.1"}, "chart-1.1.0-rc.1.tgz", "http://example.com/charts", "sha256:1234567890abc") - i.SortEntries() - r := &ChartRepository{Index: i} - - tests := []struct { - name string - chartName string - chartVersion string - wantVersion string - wantErr bool - }{ - { - name: "exact match", - chartName: "chart", - chartVersion: "0.0.1", - wantVersion: "0.0.1", - }, - { - name: "stable version", - chartName: "chart", - chartVersion: "", - wantVersion: "1.0.0", - }, - { - name: "stable version (asterisk)", - chartName: "chart", - chartVersion: "*", - wantVersion: "1.0.0", - }, - { - name: "semver range", - chartName: "chart", - chartVersion: "<1.0.0", - wantVersion: "0.2.0", - }, - { - name: "unfulfilled range", - chartName: "chart", - chartVersion: ">2.0.0", - wantErr: true, - }, - { - name: "invalid chart", - chartName: "non-existing", - wantErr: true, - }, - { - name: "match newest if ambiguous", - chartName: "chart", - chartVersion: "0.1.5", - wantVersion: "0.1.5+c.now", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cv, err := r.Get(tt.chartName, tt.chartVersion) - if (err != nil) != tt.wantErr { - t.Errorf("Get() error = %v, wantErr %v", err, tt.wantErr) - return - } - if err == nil && !strings.Contains(cv.Metadata.Version, tt.wantVersion) { - t.Errorf("Get() unexpected version = %s, want = %s", cv.Metadata.Version, tt.wantVersion) - } - }) - } -} - -func TestChartRepository_DownloadChart(t *testing.T) { - tests := []struct { - name string - url string - chartVersion *repo.ChartVersion - wantURL string - wantErr bool - }{ - { - name: "relative URL", - url: "https://example.com", - chartVersion: &repo.ChartVersion{ - Metadata: &chart.Metadata{Name: "chart"}, - URLs: []string{"charts/foo-1.0.0.tgz"}, - }, - wantURL: "https://example.com/charts/foo-1.0.0.tgz", - }, - { - name: "no chart URL", - chartVersion: &repo.ChartVersion{Metadata: &chart.Metadata{Name: "chart"}}, - wantErr: true, - }, - { - name: "invalid chart URL", - chartVersion: &repo.ChartVersion{ - Metadata: &chart.Metadata{Name: "chart"}, - URLs: []string{"https://ex ample.com/charts/foo-1.0.0.tgz"}, - }, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - mg := mockGetter{} - r := &ChartRepository{ - URL: tt.url, - Client: &mg, - } - _, err := r.DownloadChart(tt.chartVersion) - if (err != nil) != tt.wantErr { - t.Errorf("DownloadChart() error = %v, wantErr %v", err, tt.wantErr) - return - } - if err == nil && mg.requestedURL != tt.wantURL { - t.Errorf("DownloadChart() requested URL = %s, wantURL %s", mg.requestedURL, tt.wantURL) - } - }) - } -} - -func TestChartRepository_DownloadIndex(t *testing.T) { - b, err := os.ReadFile(chartmuseumtestfile) - if err != nil { - t.Fatal(err) - } - mg := mockGetter{response: b} - r := &ChartRepository{ - URL: "https://example.com", - Client: &mg, - } - if err := r.DownloadIndex(); err != nil { - t.Fatal(err) - } - if expected := r.URL + "/index.yaml"; mg.requestedURL != expected { - t.Errorf("DownloadIndex() requested URL = %s, wantURL %s", mg.requestedURL, expected) - } - verifyLocalIndex(t, r.Index) -} - -// Index load tests are derived from https://github.com/helm/helm/blob/v3.3.4/pkg/repo/index_test.go#L108 -// to ensure parity with Helm behaviour. -func TestChartRepository_LoadIndex(t *testing.T) { - tests := []struct { - name string - filename string - }{ - { - name: "regular index file", - filename: testfile, - }, - { - name: "chartmuseum index file", - filename: chartmuseumtestfile, - }, - } - - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - b, err := os.ReadFile(tt.filename) - if err != nil { - t.Fatal(err) - } - r := &ChartRepository{} - err = r.LoadIndex(b) - if err != nil { - t.Fatal(err) - } - verifyLocalIndex(t, r.Index) - }) - } -} - -func TestChartRepository_LoadIndex_Duplicates(t *testing.T) { - r := &ChartRepository{} - if err := r.LoadIndex([]byte(indexWithDuplicates)); err == nil { - t.Errorf("Expected an error when duplicate entries are present") - } -} - -func TestChartRepository_LoadIndex_Unordered(t *testing.T) { - b, err := os.ReadFile(unorderedtestfile) - if err != nil { - t.Fatal(err) - } - r := &ChartRepository{} - err = r.LoadIndex(b) - if err != nil { - t.Fatal(err) - } - verifyLocalIndex(t, r.Index) -} - -func verifyLocalIndex(t *testing.T, i *repo.IndexFile) { - numEntries := len(i.Entries) - if numEntries != 3 { - t.Errorf("Expected 3 entries in index file but got %d", numEntries) - } - - alpine, ok := i.Entries["alpine"] - if !ok { - t.Fatalf("'alpine' section not found.") - } - - if l := len(alpine); l != 1 { - t.Fatalf("'alpine' should have 1 chart, got %d", l) - } - - nginx, ok := i.Entries["nginx"] - if !ok || len(nginx) != 2 { - t.Fatalf("Expected 2 nginx entries") - } - - expects := []*repo.ChartVersion{ - { - Metadata: &chart.Metadata{ - Name: "alpine", - Description: "string", - Version: "1.0.0", - Keywords: []string{"linux", "alpine", "small", "sumtin"}, - Home: "https://github.com/something", - }, - URLs: []string{ - "https://kubernetes-charts.storage.googleapis.com/alpine-1.0.0.tgz", - "http://storage2.googleapis.com/kubernetes-charts/alpine-1.0.0.tgz", - }, - Digest: "sha256:1234567890abcdef", - }, - { - Metadata: &chart.Metadata{ - Name: "nginx", - Description: "string", - Version: "0.2.0", - Keywords: []string{"popular", "web server", "proxy"}, - Home: "https://github.com/something/else", - }, - URLs: []string{ - "https://kubernetes-charts.storage.googleapis.com/nginx-0.2.0.tgz", - }, - Digest: "sha256:1234567890abcdef", - }, - { - Metadata: &chart.Metadata{ - Name: "nginx", - Description: "string", - Version: "0.1.0", - Keywords: []string{"popular", "web server", "proxy"}, - Home: "https://github.com/something", - }, - URLs: []string{ - "https://kubernetes-charts.storage.googleapis.com/nginx-0.1.0.tgz", - }, - Digest: "sha256:1234567890abcdef", - }, - } - tests := []*repo.ChartVersion{alpine[0], nginx[0], nginx[1]} - - for i, tt := range tests { - expect := expects[i] - if tt.Name != expect.Name { - t.Errorf("Expected name %q, got %q", expect.Name, tt.Name) - } - if tt.Description != expect.Description { - t.Errorf("Expected description %q, got %q", expect.Description, tt.Description) - } - if tt.Version != expect.Version { - t.Errorf("Expected version %q, got %q", expect.Version, tt.Version) - } - if tt.Digest != expect.Digest { - t.Errorf("Expected digest %q, got %q", expect.Digest, tt.Digest) - } - if tt.Home != expect.Home { - t.Errorf("Expected home %q, got %q", expect.Home, tt.Home) - } - - for i, url := range tt.URLs { - if url != expect.URLs[i] { - t.Errorf("Expected URL %q, got %q", expect.URLs[i], url) - } - } - for i, kw := range tt.Keywords { - if kw != expect.Keywords[i] { - t.Errorf("Expected keywords %q, got %q", expect.Keywords[i], kw) - } - } - } -} - -type mockGetter struct { - requestedURL string - response []byte -} - -func (g *mockGetter) Get(url string, options ...getter.Option) (*bytes.Buffer, error) { - g.requestedURL = url - return bytes.NewBuffer(g.response), nil -} diff --git a/internal/helm/testdata/charts/empty.tgz b/internal/helm/testdata/charts/empty.tgz new file mode 100644 index 000000000..872c01559 Binary files /dev/null and b/internal/helm/testdata/charts/empty.tgz differ diff --git a/internal/helm/testdata/charts/helmchart-0.1.0.tgz b/internal/helm/testdata/charts/helmchart-0.1.0.tgz index f64a32eee..1ffdde531 100644 Binary files a/internal/helm/testdata/charts/helmchart-0.1.0.tgz and b/internal/helm/testdata/charts/helmchart-0.1.0.tgz differ diff --git a/internal/helm/testdata/charts/helmchart-v1/.helmignore b/internal/helm/testdata/charts/helmchart-v1/.helmignore new file mode 100644 index 000000000..50af03172 --- /dev/null +++ b/internal/helm/testdata/charts/helmchart-v1/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/internal/helm/testdata/charts/helmchart-v1/Chart.yaml b/internal/helm/testdata/charts/helmchart-v1/Chart.yaml new file mode 100644 index 000000000..fed8cedf2 --- /dev/null +++ b/internal/helm/testdata/charts/helmchart-v1/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A legacy Helm chart for Kubernetes +name: helmchart-v1 +version: 0.2.0 diff --git a/internal/helm/testdata/charts/helmchart-v1/templates/NOTES.txt b/internal/helm/testdata/charts/helmchart-v1/templates/NOTES.txt new file mode 100644 index 000000000..c9a8aa76a --- /dev/null +++ b/internal/helm/testdata/charts/helmchart-v1/templates/NOTES.txt @@ -0,0 +1,21 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "helmchart-v1.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "helmchart-v1.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "helmchart-v1.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "helmchart-v1.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl port-forward $POD_NAME 8080:80 +{{- end }} diff --git a/internal/helm/testdata/charts/helmchart-v1/templates/_helpers.tpl b/internal/helm/testdata/charts/helmchart-v1/templates/_helpers.tpl new file mode 100644 index 000000000..ecb988262 --- /dev/null +++ b/internal/helm/testdata/charts/helmchart-v1/templates/_helpers.tpl @@ -0,0 +1,56 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "helmchart-v1.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "helmchart-v1.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "helmchart-v1.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "helmchart-v1.labels" -}} +app.kubernetes.io/name: {{ include "helmchart-v1.name" . }} +helm.sh/chart: {{ include "helmchart-v1.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "helmchart-v1.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "helmchart-v1.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/internal/helm/testdata/charts/helmchart-v1/templates/deployment.yaml b/internal/helm/testdata/charts/helmchart-v1/templates/deployment.yaml new file mode 100644 index 000000000..8a435b3a1 --- /dev/null +++ b/internal/helm/testdata/charts/helmchart-v1/templates/deployment.yaml @@ -0,0 +1,57 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "helmchart-v1.fullname" . }} + labels: +{{ include "helmchart-v1.labels" . | indent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "helmchart-v1.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "helmchart-v1.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ template "helmchart-v1.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 80 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + readinessProbe: + httpGet: + path: / + port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/internal/helm/testdata/charts/helmchart-v1/templates/ingress.yaml b/internal/helm/testdata/charts/helmchart-v1/templates/ingress.yaml new file mode 100644 index 000000000..7db207166 --- /dev/null +++ b/internal/helm/testdata/charts/helmchart-v1/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "helmchart-v1.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: +{{ include "helmchart-v1.labels" . | indent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ . }} + backend: + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} +{{- end }} diff --git a/internal/helm/testdata/charts/helmchart-v1/templates/service.yaml b/internal/helm/testdata/charts/helmchart-v1/templates/service.yaml new file mode 100644 index 000000000..81a8cb688 --- /dev/null +++ b/internal/helm/testdata/charts/helmchart-v1/templates/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "helmchart-v1.fullname" . }} + labels: +{{ include "helmchart-v1.labels" . | indent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: {{ include "helmchart-v1.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/internal/helm/testdata/charts/helmchart-v1/templates/serviceaccount.yaml b/internal/helm/testdata/charts/helmchart-v1/templates/serviceaccount.yaml new file mode 100644 index 000000000..2f9b53dcb --- /dev/null +++ b/internal/helm/testdata/charts/helmchart-v1/templates/serviceaccount.yaml @@ -0,0 +1,8 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "helmchart-v1.serviceAccountName" . }} + labels: +{{ include "helmchart-v1.labels" . | indent 4 }} +{{- end -}} diff --git a/internal/helm/testdata/charts/helmchart-v1/templates/tests/test-connection.yaml b/internal/helm/testdata/charts/helmchart-v1/templates/tests/test-connection.yaml new file mode 100644 index 000000000..da5b5c324 --- /dev/null +++ b/internal/helm/testdata/charts/helmchart-v1/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "helmchart-v1.fullname" . }}-test-connection" + labels: +{{ include "helmchart-v1.labels" . | indent 4 }} + annotations: + "helm.sh/hook": test-success +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "helmchart-v1.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/internal/helm/testdata/charts/helmchart-v1/values.yaml b/internal/helm/testdata/charts/helmchart-v1/values.yaml new file mode 100644 index 000000000..3c03b2cd9 --- /dev/null +++ b/internal/helm/testdata/charts/helmchart-v1/values.yaml @@ -0,0 +1,68 @@ +# Default values for helmchart-v1. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: nginx + tag: stable + pullPolicy: IfNotPresent + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: [] + + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/internal/helm/testdata/charts/helmchart/values-prod.yaml b/internal/helm/testdata/charts/helmchart/values-prod.yaml new file mode 100644 index 000000000..5ef7832ca --- /dev/null +++ b/internal/helm/testdata/charts/helmchart/values-prod.yaml @@ -0,0 +1 @@ +replicaCount: 2 diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1-0.3.0.tgz b/internal/helm/testdata/charts/helmchartwithdeps-v1-0.3.0.tgz new file mode 100644 index 000000000..5b648fcfc Binary files /dev/null and b/internal/helm/testdata/charts/helmchartwithdeps-v1-0.3.0.tgz differ diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/.helmignore b/internal/helm/testdata/charts/helmchartwithdeps-v1/.helmignore new file mode 100644 index 000000000..50af03172 --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/Chart.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/Chart.yaml new file mode 100644 index 000000000..55508024f --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A legacy Helm chart for Kubernetes +name: helmchartwithdeps-v1 +version: 0.3.0 diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/requirements.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/requirements.yaml new file mode 100644 index 000000000..d6c815e6f --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/requirements.yaml @@ -0,0 +1,4 @@ +dependencies: +- name: helmchart-v1 + version: "0.2.0" + repository: "file://../helmchart-v1" diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/NOTES.txt b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/NOTES.txt new file mode 100644 index 000000000..c9a8aa76a --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/NOTES.txt @@ -0,0 +1,21 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "helmchart-v1.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "helmchart-v1.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "helmchart-v1.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "helmchart-v1.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl port-forward $POD_NAME 8080:80 +{{- end }} diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/_helpers.tpl b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/_helpers.tpl new file mode 100644 index 000000000..ecb988262 --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/_helpers.tpl @@ -0,0 +1,56 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "helmchart-v1.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "helmchart-v1.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "helmchart-v1.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "helmchart-v1.labels" -}} +app.kubernetes.io/name: {{ include "helmchart-v1.name" . }} +helm.sh/chart: {{ include "helmchart-v1.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "helmchart-v1.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "helmchart-v1.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/deployment.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/deployment.yaml new file mode 100644 index 000000000..8a435b3a1 --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/deployment.yaml @@ -0,0 +1,57 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "helmchart-v1.fullname" . }} + labels: +{{ include "helmchart-v1.labels" . | indent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "helmchart-v1.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "helmchart-v1.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ template "helmchart-v1.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 80 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + readinessProbe: + httpGet: + path: / + port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/ingress.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/ingress.yaml new file mode 100644 index 000000000..7db207166 --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "helmchart-v1.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: +{{ include "helmchart-v1.labels" . | indent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ . }} + backend: + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} +{{- end }} diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/service.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/service.yaml new file mode 100644 index 000000000..81a8cb688 --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "helmchart-v1.fullname" . }} + labels: +{{ include "helmchart-v1.labels" . | indent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: {{ include "helmchart-v1.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/serviceaccount.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/serviceaccount.yaml new file mode 100644 index 000000000..2f9b53dcb --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/serviceaccount.yaml @@ -0,0 +1,8 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "helmchart-v1.serviceAccountName" . }} + labels: +{{ include "helmchart-v1.labels" . | indent 4 }} +{{- end -}} diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/tests/test-connection.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/tests/test-connection.yaml new file mode 100644 index 000000000..da5b5c324 --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "helmchart-v1.fullname" . }}-test-connection" + labels: +{{ include "helmchart-v1.labels" . | indent 4 }} + annotations: + "helm.sh/hook": test-success +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "helmchart-v1.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/values.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/values.yaml new file mode 100644 index 000000000..3c03b2cd9 --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/values.yaml @@ -0,0 +1,68 @@ +# Default values for helmchart-v1. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: nginx + tag: stable + pullPolicy: IfNotPresent + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: [] + + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/internal/helm/testdata/charts/helmchartwithdeps/Chart.lock b/internal/helm/testdata/charts/helmchartwithdeps/Chart.lock new file mode 100644 index 000000000..83401ac65 --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps/Chart.lock @@ -0,0 +1,12 @@ +dependencies: +- name: helmchart + repository: file://../helmchart + version: 0.1.0 +- name: helmchart + repository: file://../helmchart + version: 0.1.0 +- name: grafana + repository: https://grafana.github.io/helm-charts + version: 6.17.4 +digest: sha256:1e41c97e27347f433ff0212bf52c344bc82dd435f70129d15e96cd2c8fcc32bb +generated: "2021-11-02T01:25:59.624290788+01:00" diff --git a/main.go b/main.go index 55a2d2f97..7853f224b 100644 --- a/main.go +++ b/main.go @@ -45,6 +45,7 @@ import ( sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" "github.com/fluxcd/source-controller/controllers" + "github.com/fluxcd/source-controller/internal/helm" // +kubebuilder:scaffold:imports ) @@ -79,6 +80,9 @@ func main() { concurrent int requeueDependency time.Duration watchAllNamespaces bool + helmIndexLimit int64 + helmChartLimit int64 + helmChartFileLimit int64 clientOptions client.Options logOptions logger.Options leaderElectionOptions leaderelection.Options @@ -98,7 +102,15 @@ func main() { flag.IntVar(&concurrent, "concurrent", 2, "The number of concurrent reconciles per controller.") flag.BoolVar(&watchAllNamespaces, "watch-all-namespaces", true, "Watch for custom resources in all namespaces, if set to false it will only watch the runtime namespace.") - flag.DurationVar(&requeueDependency, "requeue-dependency", 30*time.Second, "The interval at which failing dependencies are reevaluated.") + flag.Int64Var(&helmIndexLimit, "helm-index-max-size", helm.MaxIndexSize, + "The max allowed size in bytes of a Helm repository index file.") + flag.Int64Var(&helmChartLimit, "helm-chart-max-size", helm.MaxChartSize, + "The max allowed size in bytes of a Helm chart file.") + flag.Int64Var(&helmChartFileLimit, "helm-chart-file-max-size", helm.MaxChartFileSize, + "The max allowed size in bytes of a file in a Helm chart.") + flag.DurationVar(&requeueDependency, "requeue-dependency", 30*time.Second, + "The interval at which failing dependencies are reevaluated.") + clientOptions.BindFlags(flag.CommandLine) logOptions.BindFlags(flag.CommandLine) leaderElectionOptions.BindFlags(flag.CommandLine) @@ -106,6 +118,11 @@ func main() { ctrl.SetLogger(logger.NewLogger(logOptions)) + // Set upper bound file size limits Helm + helm.MaxIndexSize = helmIndexLimit + helm.MaxChartSize = helmChartLimit + helm.MaxChartFileSize = helmChartFileLimit + var eventRecorder *events.Recorder if eventsAddr != "" { if er, err := events.NewRecorder(eventsAddr, controllerName); err != nil {