From 8537a0f8fab4242f960a6c867bc781965a3bb78f Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Fri, 29 Oct 2021 14:22:44 +0200 Subject: [PATCH 01/23] internal/helm: add helpers to load chart metadata This commits adds `LoadChartMetadataFromArchive` and `LoadChartMetadataFromDir` helpers to the internal `helm` package to be able to make observations to the Helm metadata file without loading the chart in full. The helpers are compatible with charts of the v1 format (with a separate `requirements.yaml` file), and an additional `LoadChartMetadata` helper is available to automatically call the right `LoadChartMetadataFrom*` version by looking at the file description of the given path. Signed-off-by: Hidde Beydals --- internal/helm/chart.go | 137 +++++++++++++++++- internal/helm/chart_test.go | 112 +++++++++++++- internal/helm/dependency_manager_test.go | 11 +- internal/helm/testdata/charts/empty.tgz | Bin 0 -> 45 bytes .../testdata/charts/helmchart-v1/.helmignore | 22 +++ .../testdata/charts/helmchart-v1/Chart.yaml | 5 + .../charts/helmchart-v1/templates/NOTES.txt | 21 +++ .../helmchart-v1/templates/_helpers.tpl | 56 +++++++ .../helmchart-v1/templates/deployment.yaml | 57 ++++++++ .../helmchart-v1/templates/ingress.yaml | 41 ++++++ .../helmchart-v1/templates/service.yaml | 16 ++ .../templates/serviceaccount.yaml | 8 + .../templates/tests/test-connection.yaml | 15 ++ .../testdata/charts/helmchart-v1/values.yaml | 68 +++++++++ .../charts/helmchartwithdeps-v1-0.3.0.tgz | Bin 0 -> 3845 bytes .../charts/helmchartwithdeps-v1/.helmignore | 22 +++ .../charts/helmchartwithdeps-v1/Chart.yaml | 5 + .../helmchartwithdeps-v1/requirements.yaml | 4 + .../helmchartwithdeps-v1/templates/NOTES.txt | 21 +++ .../templates/_helpers.tpl | 56 +++++++ .../templates/deployment.yaml | 57 ++++++++ .../templates/ingress.yaml | 41 ++++++ .../templates/service.yaml | 16 ++ .../templates/serviceaccount.yaml | 8 + .../templates/tests/test-connection.yaml | 15 ++ .../charts/helmchartwithdeps-v1/values.yaml | 68 +++++++++ 26 files changed, 875 insertions(+), 7 deletions(-) create mode 100644 internal/helm/testdata/charts/empty.tgz create mode 100644 internal/helm/testdata/charts/helmchart-v1/.helmignore create mode 100644 internal/helm/testdata/charts/helmchart-v1/Chart.yaml create mode 100644 internal/helm/testdata/charts/helmchart-v1/templates/NOTES.txt create mode 100644 internal/helm/testdata/charts/helmchart-v1/templates/_helpers.tpl create mode 100644 internal/helm/testdata/charts/helmchart-v1/templates/deployment.yaml create mode 100644 internal/helm/testdata/charts/helmchart-v1/templates/ingress.yaml create mode 100644 internal/helm/testdata/charts/helmchart-v1/templates/service.yaml create mode 100644 internal/helm/testdata/charts/helmchart-v1/templates/serviceaccount.yaml create mode 100644 internal/helm/testdata/charts/helmchart-v1/templates/tests/test-connection.yaml create mode 100644 internal/helm/testdata/charts/helmchart-v1/values.yaml create mode 100644 internal/helm/testdata/charts/helmchartwithdeps-v1-0.3.0.tgz create mode 100644 internal/helm/testdata/charts/helmchartwithdeps-v1/.helmignore create mode 100644 internal/helm/testdata/charts/helmchartwithdeps-v1/Chart.yaml create mode 100644 internal/helm/testdata/charts/helmchartwithdeps-v1/requirements.yaml create mode 100644 internal/helm/testdata/charts/helmchartwithdeps-v1/templates/NOTES.txt create mode 100644 internal/helm/testdata/charts/helmchartwithdeps-v1/templates/_helpers.tpl create mode 100644 internal/helm/testdata/charts/helmchartwithdeps-v1/templates/deployment.yaml create mode 100644 internal/helm/testdata/charts/helmchartwithdeps-v1/templates/ingress.yaml create mode 100644 internal/helm/testdata/charts/helmchartwithdeps-v1/templates/service.yaml create mode 100644 internal/helm/testdata/charts/helmchartwithdeps-v1/templates/serviceaccount.yaml create mode 100644 internal/helm/testdata/charts/helmchartwithdeps-v1/templates/tests/test-connection.yaml create mode 100644 internal/helm/testdata/charts/helmchartwithdeps-v1/values.yaml diff --git a/internal/helm/chart.go b/internal/helm/chart.go index 6630f4f74..accbc69a9 100644 --- a/internal/helm/chart.go +++ b/internal/helm/chart.go @@ -17,15 +17,24 @@ limitations under the License. package helm import ( + "archive/tar" + "bufio" + "compress/gzip" + "errors" "fmt" + "io" + "os" + "path" + "path/filepath" "reflect" + "strings" helmchart "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chartutil" + "sigs.k8s.io/yaml" ) -// OverwriteChartDefaultValues overwrites the chart default values file with the -// given data. +// OverwriteChartDefaultValues overwrites the chart default values file with the given data. func OverwriteChartDefaultValues(chart *helmchart.Chart, data []byte) (bool, error) { // Read override values file data values, err := chartutil.ReadValues(data) @@ -57,3 +66,127 @@ func OverwriteChartDefaultValues(chart *helmchart.Chart, data []byte) (bool, err // This should never happen, helm charts must have a values.yaml file to be valid return false, fmt.Errorf("failed to locate values file: %s", chartutil.ValuesfileName) } + +// LoadChartMetadata attempts to load the chart.Metadata from the "Chart.yaml" file in the directory or archive at the +// given chartPath. It takes "requirements.yaml" files into account, and is therefore compatible with the +// chart.APIVersionV1 format. +func LoadChartMetadata(chartPath string) (*helmchart.Metadata, error) { + i, err := os.Stat(chartPath) + if err != nil { + return nil, err + } + switch { + case i.IsDir(): + return LoadChartMetadataFromDir(chartPath) + default: + return LoadChartMetadataFromArchive(chartPath) + } +} + +// LoadChartMetadataFromDir loads the chart.Metadata from the "Chart.yaml" file in the directory at the given path. +// It takes "requirements.yaml" files into account, and is therefore compatible with the chart.APIVersionV1 format. +func LoadChartMetadataFromDir(dir string) (*helmchart.Metadata, error) { + m := new(helmchart.Metadata) + + b, err := os.ReadFile(filepath.Join(dir, chartutil.ChartfileName)) + if err != nil { + return nil, err + } + err = yaml.Unmarshal(b, m) + if err != nil { + return nil, fmt.Errorf("cannot load '%s': %w", chartutil.ChartfileName, err) + } + if m.APIVersion == "" { + m.APIVersion = helmchart.APIVersionV1 + } + + b, err = os.ReadFile(filepath.Join(dir, "requirements.yaml")) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, err + } + if len(b) > 0 { + if err = yaml.Unmarshal(b, m); err != nil { + return nil, fmt.Errorf("cannot load 'requirements.yaml': %w", err) + } + } + return m, nil +} + +// LoadChartMetadataFromArchive loads the chart.Metadata from the "Chart.yaml" file in the archive at the given path. +// It takes "requirements.yaml" files into account, and is therefore compatible with the chart.APIVersionV1 format. +func LoadChartMetadataFromArchive(archive string) (*helmchart.Metadata, error) { + f, err := os.Open(archive) + if err != nil { + return nil, err + } + defer f.Close() + + r := bufio.NewReader(f) + zr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + tr := tar.NewReader(zr) + + var m *helmchart.Metadata + for { + hd, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hd.FileInfo().IsDir() { + // Use this instead of hd.Typeflag because we don't have to do any + // inference chasing. + continue + } + + switch hd.Typeflag { + // We don't want to process these extension header files. + case tar.TypeXGlobalHeader, tar.TypeXHeader: + continue + } + + // Archive could contain \ if generated on Windows + delimiter := "/" + if strings.ContainsRune(hd.Name, '\\') { + delimiter = "\\" + } + parts := strings.Split(hd.Name, delimiter) + + // We are only interested in files in the base directory + if len(parts) != 2 { + continue + } + + // Normalize the path to the / delimiter + n := strings.Join(parts[1:], delimiter) + n = strings.ReplaceAll(n, delimiter, "/") + n = path.Clean(n) + + switch parts[1] { + case chartutil.ChartfileName, "requirements.yaml": + b, err := io.ReadAll(tr) + if err != nil { + return nil, err + } + if m == nil { + m = new(helmchart.Metadata) + } + err = yaml.Unmarshal(b, m) + if err != nil { + return nil, fmt.Errorf("cannot load '%s': %w", parts[1], err) + } + if m.APIVersion == "" { + m.APIVersion = helmchart.APIVersionV1 + } + } + } + if m == nil { + return nil, fmt.Errorf("no '%s' found", chartutil.ChartfileName) + } + return m, nil +} diff --git a/internal/helm/chart_test.go b/internal/helm/chart_test.go index c0b3e8c58..7afa2a3f6 100644 --- a/internal/helm/chart_test.go +++ b/internal/helm/chart_test.go @@ -20,19 +20,20 @@ import ( "reflect" "testing" + . "github.com/onsi/gomega" helmchart "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chartutil" ) var ( - originalValuesFixture []byte = []byte("override: original") - chartFilesFixture []*helmchart.File = []*helmchart.File{ + originalValuesFixture = []byte("override: original") + chartFilesFixture = []*helmchart.File{ { Name: "values.yaml", Data: originalValuesFixture, }, } - chartFixture helmchart.Chart = helmchart.Chart{ + chartFixture = helmchart.Chart{ Metadata: &helmchart.Metadata{ Name: "test", Version: "0.1.0", @@ -111,3 +112,108 @@ func TestOverwriteChartDefaultValues(t *testing.T) { }) } } + +func Test_LoadChartMetadataFromDir(t *testing.T) { + tests := []struct { + name string + dir string + wantName string + wantVersion string + wantDependencyCount int + wantErr string + }{ + { + name: "Loads from dir", + dir: "testdata/charts/helmchart", + wantName: "helmchart", + wantVersion: "0.1.0", + }, + { + name: "Loads from v1 dir including requirements.yaml", + dir: "testdata/charts/helmchartwithdeps-v1", + wantName: chartNameV1, + wantVersion: chartVersionV1, + wantDependencyCount: 1, + }, + { + name: "Error if no Chart.yaml", + dir: "testdata/charts/", + wantErr: "testdata/charts/Chart.yaml: no such file or directory", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + got, err := LoadChartMetadataFromDir(tt.dir) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(got).To(BeNil()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).ToNot(BeNil()) + g.Expect(got.Validate()).To(Succeed()) + g.Expect(got.Name).To(Equal(tt.wantName)) + g.Expect(got.Version).To(Equal(tt.wantVersion)) + g.Expect(got.Dependencies).To(HaveLen(tt.wantDependencyCount)) + }) + } +} + +func TestLoadChartMetadataFromArchive(t *testing.T) { + tests := []struct { + name string + archive string + wantName string + wantVersion string + wantDependencyCount int + wantErr string + }{ + { + name: "Loads from archive", + archive: helmPackageFile, + wantName: chartName, + wantVersion: chartVersion, + }, + { + name: "Loads from v1 archive including requirements.yaml", + archive: helmPackageV1File, + wantName: chartNameV1, + wantVersion: chartVersionV1, + wantDependencyCount: 1, + }, + { + name: "Error on not found", + archive: "testdata/invalid.tgz", + wantErr: "no such file or directory", + }, + { + name: "Error if no Chart.yaml", + archive: "testdata/charts/empty.tgz", + wantErr: "no 'Chart.yaml' found", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + got, err := LoadChartMetadataFromArchive(tt.archive) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(got).To(BeNil()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).ToNot(BeNil()) + g.Expect(got.Validate()).To(Succeed()) + g.Expect(got.Name).To(Equal(tt.wantName)) + g.Expect(got.Version).To(Equal(tt.wantVersion)) + g.Expect(got.Dependencies).To(HaveLen(tt.wantDependencyCount)) + }) + } +} diff --git a/internal/helm/dependency_manager_test.go b/internal/helm/dependency_manager_test.go index 5a5def3c2..6a38997b2 100644 --- a/internal/helm/dependency_manager_test.go +++ b/internal/helm/dependency_manager_test.go @@ -28,8 +28,9 @@ import ( ) var ( - helmPackageFile = "testdata/charts/helmchart-0.1.0.tgz" - + // helmPackageFile contains the path to a Helm package in the v2 format + // without any dependencies + helmPackageFile = "testdata/charts/helmchart-0.1.0.tgz" chartName = "helmchart" chartVersion = "0.1.0" chartLocalRepository = "file://../helmchart" @@ -38,6 +39,12 @@ var ( Version: chartVersion, Repository: "https://example.com/charts", } + // helmPackageV1File contains the path to a Helm package in the v1 format, + // including dependencies in a requirements.yaml file which should be + // loaded + helmPackageV1File = "testdata/charts/helmchartwithdeps-v1-0.3.0.tgz" + chartNameV1 = "helmchartwithdeps-v1" + chartVersionV1 = "0.3.0" ) func TestBuild_WithEmptyDependencies(t *testing.T) { diff --git a/internal/helm/testdata/charts/empty.tgz b/internal/helm/testdata/charts/empty.tgz new file mode 100644 index 0000000000000000000000000000000000000000..872c01559ea7ac2302b7d73ae653f7ad29f8981d GIT binary patch literal 45 qcmb2|=3oE==C=nKd4a4$3%>j3@|iDyhyuw5bLLt1WYrlo7#IN4*9i&$ literal 0 HcmV?d00001 diff --git a/internal/helm/testdata/charts/helmchart-v1/.helmignore b/internal/helm/testdata/charts/helmchart-v1/.helmignore new file mode 100644 index 000000000..50af03172 --- /dev/null +++ b/internal/helm/testdata/charts/helmchart-v1/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/internal/helm/testdata/charts/helmchart-v1/Chart.yaml b/internal/helm/testdata/charts/helmchart-v1/Chart.yaml new file mode 100644 index 000000000..fed8cedf2 --- /dev/null +++ b/internal/helm/testdata/charts/helmchart-v1/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A legacy Helm chart for Kubernetes +name: helmchart-v1 +version: 0.2.0 diff --git a/internal/helm/testdata/charts/helmchart-v1/templates/NOTES.txt b/internal/helm/testdata/charts/helmchart-v1/templates/NOTES.txt new file mode 100644 index 000000000..c9a8aa76a --- /dev/null +++ b/internal/helm/testdata/charts/helmchart-v1/templates/NOTES.txt @@ -0,0 +1,21 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "helmchart-v1.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "helmchart-v1.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "helmchart-v1.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "helmchart-v1.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl port-forward $POD_NAME 8080:80 +{{- end }} diff --git a/internal/helm/testdata/charts/helmchart-v1/templates/_helpers.tpl b/internal/helm/testdata/charts/helmchart-v1/templates/_helpers.tpl new file mode 100644 index 000000000..ecb988262 --- /dev/null +++ b/internal/helm/testdata/charts/helmchart-v1/templates/_helpers.tpl @@ -0,0 +1,56 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "helmchart-v1.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "helmchart-v1.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "helmchart-v1.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "helmchart-v1.labels" -}} +app.kubernetes.io/name: {{ include "helmchart-v1.name" . }} +helm.sh/chart: {{ include "helmchart-v1.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "helmchart-v1.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "helmchart-v1.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/internal/helm/testdata/charts/helmchart-v1/templates/deployment.yaml b/internal/helm/testdata/charts/helmchart-v1/templates/deployment.yaml new file mode 100644 index 000000000..8a435b3a1 --- /dev/null +++ b/internal/helm/testdata/charts/helmchart-v1/templates/deployment.yaml @@ -0,0 +1,57 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "helmchart-v1.fullname" . }} + labels: +{{ include "helmchart-v1.labels" . | indent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "helmchart-v1.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "helmchart-v1.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ template "helmchart-v1.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 80 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + readinessProbe: + httpGet: + path: / + port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/internal/helm/testdata/charts/helmchart-v1/templates/ingress.yaml b/internal/helm/testdata/charts/helmchart-v1/templates/ingress.yaml new file mode 100644 index 000000000..7db207166 --- /dev/null +++ b/internal/helm/testdata/charts/helmchart-v1/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "helmchart-v1.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: +{{ include "helmchart-v1.labels" . | indent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ . }} + backend: + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} +{{- end }} diff --git a/internal/helm/testdata/charts/helmchart-v1/templates/service.yaml b/internal/helm/testdata/charts/helmchart-v1/templates/service.yaml new file mode 100644 index 000000000..81a8cb688 --- /dev/null +++ b/internal/helm/testdata/charts/helmchart-v1/templates/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "helmchart-v1.fullname" . }} + labels: +{{ include "helmchart-v1.labels" . | indent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: {{ include "helmchart-v1.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/internal/helm/testdata/charts/helmchart-v1/templates/serviceaccount.yaml b/internal/helm/testdata/charts/helmchart-v1/templates/serviceaccount.yaml new file mode 100644 index 000000000..2f9b53dcb --- /dev/null +++ b/internal/helm/testdata/charts/helmchart-v1/templates/serviceaccount.yaml @@ -0,0 +1,8 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "helmchart-v1.serviceAccountName" . }} + labels: +{{ include "helmchart-v1.labels" . | indent 4 }} +{{- end -}} diff --git a/internal/helm/testdata/charts/helmchart-v1/templates/tests/test-connection.yaml b/internal/helm/testdata/charts/helmchart-v1/templates/tests/test-connection.yaml new file mode 100644 index 000000000..da5b5c324 --- /dev/null +++ b/internal/helm/testdata/charts/helmchart-v1/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "helmchart-v1.fullname" . }}-test-connection" + labels: +{{ include "helmchart-v1.labels" . | indent 4 }} + annotations: + "helm.sh/hook": test-success +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "helmchart-v1.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/internal/helm/testdata/charts/helmchart-v1/values.yaml b/internal/helm/testdata/charts/helmchart-v1/values.yaml new file mode 100644 index 000000000..3c03b2cd9 --- /dev/null +++ b/internal/helm/testdata/charts/helmchart-v1/values.yaml @@ -0,0 +1,68 @@ +# Default values for helmchart-v1. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: nginx + tag: stable + pullPolicy: IfNotPresent + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: [] + + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1-0.3.0.tgz b/internal/helm/testdata/charts/helmchartwithdeps-v1-0.3.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..5b648fcfcf640ddcdf5b48563cabe20678e2d7fa GIT binary patch literal 3845 zcmY+HcTm&I-o_CSLDWzLDJn`2O^~Lv(53fYq<0KWr36F~B9YL03rY(_1nE6FaHIU@B1;9WzNc9YZ;T@Q1Rl z(pHl0=CVF^<`3nJt*qn>t^Ivnpzh%orUA-29{x|Lek(@@5L(jC?2BBDHZ+=}8pX)g zb<`26_ck^z*5uO}I`Q0QPl(Ctm6MmL6aWjU1Z>vx=;$_y5mM6|MkFxgzf(PYAKVOn z5eUySNHHv7NIibLri(=Q4#ylITIYucGL*?&KTojH-?j)i?3Oif++}*Gi)UT58sQQg~HajycPUW8O$I-b~{ks)7C_SF@J9zQ1rG-$0n1IZ%u?1$4o9UOY)>P{utS+7sR?Y*+tR2&x;+V5!Q+baa?sG}o-H zZiM*3@pt@vnX5$C&+(c$%(??9@2SO|<5QT*B2fHoDjCtSgUR^O*2|-+L9Z4-f zBORUjO+PTbAb%Gr14(D3^>W0lE0ko^CgvxpKP+L!B6#)FY5rW7U_q|L7sq^HSYyz; zlhRC+CQB>J3{Dt+$%oK%`>pzdEhYuZ0@@r_aAyZ3al+`i9JocY^CbPagr@H^qzc(lTh`PkWOc}!LKx(|^666NE zRasQ3lg^}o-QtG&2e7BcUICZn6%U5dGU=yYpAZk*rF3>?=`4T>h9Q^V1NrwYLdt^d z77zwx?q`B6J|>-K#&P#!?+Eqm4DP*KW`GBFo0XZOQ2C8_O0UgNhTa_PW62b_+-eIB zqUY<*=+b*|BacJ6d?z&rC6NsuDk(06Y-u5d8|@v)E>F6+2#X=FMnQ=BCTJ z=i#eNSh{C3SUxDl0Ox|G1V3N~<#pD>DW~F-whn{P=^tkXm`R(`7&YP@*&vh zV+5l@ALMIfJPuuP5Y;`;+so+A;d6K6w}PEy31b}_G>e@xQ5YQojV8Vanx}!XEND^G z(cWhF*NNl1a^=00&@*U_D{$#i6s{I{^YX9PMUv?ZX*Mu_cBa!^5{zRc(R*j6N|=^U zHMURWDSHAto;j>Y`cW-DzjZt=pPQi0qM{w<`IIfTM&8c;mgh8el^x7V)-zPdPADO6 zb;WkycG!Ts(dOltlf1l3d#8=NqisiENWIO==5Ca12qxR$eHRZyHN)cN8EIxxYs$1^ zT0Y_SCmg}+yH-X!aN@MSd#3$)%RQ03LX>rd zbj$Zn?@}&wmQoqQDD%<6i8*%w*q%qsRfewWtg7+HWDF1>R7 zgh>#Uws>41#zk-G-GBh~E9S>t89NrsMpd@s9vOiVC|H@hwy?D5_o3n1xafrIyxsF} zSG?20y`LBKI*B5>cNL8r^R)17)1lNwHG+Q@X8T6bLiS5|*BSJ>@=2`K2NUv73HD9b zRDBXCqc7}g53-&`k~MdU=30Pgfr3UCvRfbPI1t!iUr`-RW4vl@{>)2Q#aB`gWI3O6t4Xqi{fh1WW6%) zLGWEX&|jW?=6d)l^8wsZ8I|7bmpjY1NZ5Dt`~Dd!*(o;WZ0YiTV#t*9h^E0Hoc48f zXX)33r!vJ8Kbx%jk#+YcI8%jeD`Le?B(tE`aOl`E=AKD!Eq(0rDWml7rcSvij$pPc zQoPy-5vd+CcSY{Kc-{myCd3pSXrp%$iiqYI5!WE19CMm_Z6%?zjH3^4z54R)1>D_d zvW`B2{mWRO7N;~t7jvc1^YxSX(aO$3oG9$^r?Out>(`D$a*Eu)dqUmH67G%u=IEX{ zF@Ay=5m3A@wk{Z~yFNXu$eN=wxKBTWtNbySVc(Ut56kXpT90_boaH$|Z-k;qO8a>| zaFVbU)rR-4c}_iUQS7tu41Zp@9Q+fMk|Rl)Yl#l+I$Izak7q#=00IfPL2_C{1iqHFAAVJE-RYi-ELZWJMG9q-@e!h!GH5}=nF{`S)%Ou~4$ zE?hS9N;S)mu#E`Bsy z4h&(1o>mQTK%p}t<-2AIQKVeU>Zn3G?&`-p=3N8G|` zrx$pih3I5j+1weA$iAazUg1bIao1j@+%t9TaB*+Tt{2^~Z1JG*y%O`+SRL9!h|{IV zct5g4m{1wbBO+w{?$YfKhpgXwMididsv>QZj`jNW2m6SNK}{hbc!lmK?5G$dW`e7= z)nfgLv}M}E9yDik6YR67pz@d9*`+s&=j+jv(J02_7YrRs>P)lB3w!V|IIZx5ic_0pR4lwLd?~1 z|L$RM`vtcAhVYjzveRGuY;Nh=bfsu!_meRBsrmy`NsWzVK1=Z4tZ&uQ+{lXDGb^v1x3Tn7iK;thW zLlNP+7BR9N+XCa*B4SDgj%GN0e>s(mPkD*56u7xR=L)r?X*__eSpUL(<$lUdhS2)6 zt%ooNq8`y$&4y69bU zFk};ldm3j4NEkg%N#3vsu%lI4w27m_Db1xbG#3OlJ`l)zhE8~Q!Z~810w0+c0Z^^_ zeGWROTv6W!qH)p{SYT=$#C4vp6c6#m zT^s3eejStP(0Zjl7;Jk{oe?o&-Vfr9QfYmeqqi95h9MkC^UZC;0a3z1eTx)l^YEi> z%i%CsoVQN#kW8%?Pv-8pxiw;UoI6c_xeSd)?Ct;s=s&qSB4!uS0dwF!{nDeWd>dgh z+T$bBy4(4rVGDj&e;}?P`Qy&L?ZXVchMI+N1I=vEdI=Gz={=IR?8|b8DwZ z;FUOv>B?aTc8(zlrsnl06vujuJ-LzA)fXEGH0GD_F z8Sq_$sQtg_ydM=B2m-c7WZ$3P(22*!h0vO9;2O>UdY#K;_lvNQmN!7?@;Mk~PWue~ z7=H?&|5p7kt8>l27+avtd0@6x&|;QcbM9$)y5)@j7X7cwxgzo7smlEA1K^)fA~0ET z&Uer@3;@dXGW$irQN(ljlG80GR*VrvejT!(l$3Xf15uwNZE5 Dk3PG3 literal 0 HcmV?d00001 diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/.helmignore b/internal/helm/testdata/charts/helmchartwithdeps-v1/.helmignore new file mode 100644 index 000000000..50af03172 --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/Chart.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/Chart.yaml new file mode 100644 index 000000000..55508024f --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A legacy Helm chart for Kubernetes +name: helmchartwithdeps-v1 +version: 0.3.0 diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/requirements.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/requirements.yaml new file mode 100644 index 000000000..d6c815e6f --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/requirements.yaml @@ -0,0 +1,4 @@ +dependencies: +- name: helmchart-v1 + version: "0.2.0" + repository: "file://../helmchart-v1" diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/NOTES.txt b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/NOTES.txt new file mode 100644 index 000000000..c9a8aa76a --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/NOTES.txt @@ -0,0 +1,21 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "helmchart-v1.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "helmchart-v1.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "helmchart-v1.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "helmchart-v1.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl port-forward $POD_NAME 8080:80 +{{- end }} diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/_helpers.tpl b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/_helpers.tpl new file mode 100644 index 000000000..ecb988262 --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/_helpers.tpl @@ -0,0 +1,56 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "helmchart-v1.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "helmchart-v1.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "helmchart-v1.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "helmchart-v1.labels" -}} +app.kubernetes.io/name: {{ include "helmchart-v1.name" . }} +helm.sh/chart: {{ include "helmchart-v1.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "helmchart-v1.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "helmchart-v1.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/deployment.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/deployment.yaml new file mode 100644 index 000000000..8a435b3a1 --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/deployment.yaml @@ -0,0 +1,57 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "helmchart-v1.fullname" . }} + labels: +{{ include "helmchart-v1.labels" . | indent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "helmchart-v1.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "helmchart-v1.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ template "helmchart-v1.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 80 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + readinessProbe: + httpGet: + path: / + port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/ingress.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/ingress.yaml new file mode 100644 index 000000000..7db207166 --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "helmchart-v1.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: +{{ include "helmchart-v1.labels" . | indent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: +{{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ . }} + backend: + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} +{{- end }} diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/service.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/service.yaml new file mode 100644 index 000000000..81a8cb688 --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "helmchart-v1.fullname" . }} + labels: +{{ include "helmchart-v1.labels" . | indent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: {{ include "helmchart-v1.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/serviceaccount.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/serviceaccount.yaml new file mode 100644 index 000000000..2f9b53dcb --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/serviceaccount.yaml @@ -0,0 +1,8 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "helmchart-v1.serviceAccountName" . }} + labels: +{{ include "helmchart-v1.labels" . | indent 4 }} +{{- end -}} diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/tests/test-connection.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/tests/test-connection.yaml new file mode 100644 index 000000000..da5b5c324 --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "helmchart-v1.fullname" . }}-test-connection" + labels: +{{ include "helmchart-v1.labels" . | indent 4 }} + annotations: + "helm.sh/hook": test-success +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "helmchart-v1.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/internal/helm/testdata/charts/helmchartwithdeps-v1/values.yaml b/internal/helm/testdata/charts/helmchartwithdeps-v1/values.yaml new file mode 100644 index 000000000..3c03b2cd9 --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps-v1/values.yaml @@ -0,0 +1,68 @@ +# Default values for helmchart-v1. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: nginx + tag: stable + pullPolicy: IfNotPresent + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: [] + + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} From 44c18633349f867f71ba3f20d0e57a4f734bfe36 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Sat, 30 Oct 2021 01:27:04 +0200 Subject: [PATCH 02/23] internal/helm: add repository cache helpers This commits adds simple caching capabilities to the `ChartRepository`, which makes it possible to load the `Index` from a defined `CachePath` using `LoadFromCache()`, and to download the index to a new `CachePath` using `CacheIndex()`. In addition, the repository tests have been updated to make use of Gomega, and some missing ones have been added. Signed-off-by: Hidde Beydals --- internal/helm/repository.go | 156 +++++++++-- internal/helm/repository_test.go | 443 ++++++++++++++++++++----------- internal/helm/utils_test.go | 60 +++++ 3 files changed, 474 insertions(+), 185 deletions(-) create mode 100644 internal/helm/utils_test.go diff --git a/internal/helm/repository.go b/internal/helm/repository.go index 49728452d..c57df111f 100644 --- a/internal/helm/repository.go +++ b/internal/helm/repository.go @@ -18,12 +18,17 @@ package helm import ( "bytes" + "crypto/sha256" + "encoding/hex" + "errors" "fmt" "io" "net/url" + "os" "path" "sort" "strings" + "sync" "github.com/Masterminds/semver/v3" "helm.sh/helm/v3/pkg/getter" @@ -33,20 +38,37 @@ import ( "github.com/fluxcd/pkg/version" ) +var ErrNoChartIndex = errors.New("no chart index") + // ChartRepository represents a Helm chart repository, and the configuration -// required to download the chart index, and charts from the repository. +// required to download the chart index and charts from the repository. +// All methods are thread safe unless defined otherwise. type ChartRepository struct { - URL string - Index *repo.IndexFile - Client getter.Getter + // URL the ChartRepository's index.yaml can be found at, + // without the index.yaml suffix. + URL string + // Client to use while downloading the Index or a chart from the URL. + Client getter.Getter + // Options to configure the Client with while downloading the Index + // or a chart from the URL. Options []getter.Option + // CachePath is the path of a cached index.yaml for read-only operations. + CachePath string + // Index contains a loaded chart repository index if not nil. + Index *repo.IndexFile + // Checksum contains the SHA256 checksum of the loaded chart repository + // index bytes. + Checksum string + + *sync.RWMutex } // NewChartRepository constructs and returns a new ChartRepository with // the ChartRepository.Client configured to the getter.Getter for the // repository URL scheme. It returns an error on URL parsing failures, // or if there is no getter available for the scheme. -func NewChartRepository(repositoryURL string, providers getter.Providers, opts []getter.Option) (*ChartRepository, error) { +func NewChartRepository(repositoryURL, cachePath string, providers getter.Providers, opts []getter.Option) (*ChartRepository, error) { + r := newChartRepository() u, err := url.Parse(repositoryURL) if err != nil { return nil, err @@ -55,17 +77,29 @@ func NewChartRepository(repositoryURL string, providers getter.Providers, opts [ if err != nil { return nil, err } + r.URL = repositoryURL + r.CachePath = cachePath + r.Client = c + r.Options = opts + return r, nil +} + +func newChartRepository() *ChartRepository { return &ChartRepository{ - URL: repositoryURL, - Client: c, - Options: opts, - }, nil + RWMutex: &sync.RWMutex{}, + } } // Get returns the repo.ChartVersion for the given name, the version is expected // to be a semver.Constraints compatible string. If version is empty, the latest // stable version will be returned and prerelease versions will be ignored. func (r *ChartRepository) Get(name, ver string) (*repo.ChartVersion, error) { + r.RLock() + defer r.RUnlock() + + if r.Index == nil { + return nil, ErrNoChartIndex + } cvs, ok := r.Index.Entries[name] if !ok { return nil, repo.ErrNoChartName @@ -114,7 +148,7 @@ func (r *ChartRepository) Get(name, ver string) (*repo.ChartVersion, error) { lookup[v] = cv } if len(matchedVersions) == 0 { - return nil, fmt.Errorf("no chart version found for %s-%s", name, ver) + return nil, fmt.Errorf("no '%s' chart with version matching '%s' found", name, ver) } // Sort versions @@ -145,7 +179,7 @@ func (r *ChartRepository) Get(name, ver string) (*repo.ChartVersion, error) { // ChartRepository. It returns a bytes.Buffer containing the chart data. func (r *ChartRepository) DownloadChart(chart *repo.ChartVersion) (*bytes.Buffer, error) { if len(chart.URLs) == 0 { - return nil, fmt.Errorf("chart %q has no downloadable URLs", chart.Name) + return nil, fmt.Errorf("chart '%s' has no downloadable URLs", chart.Name) } // TODO(hidde): according to the Helm source the first item is not @@ -175,13 +209,9 @@ func (r *ChartRepository) DownloadChart(chart *repo.ChartVersion) (*bytes.Buffer return r.Client.Get(u.String(), r.Options...) } -// LoadIndex loads the given bytes into the Index while performing -// minimal validity checks. It fails if the API version is not set -// (repo.ErrNoAPIVersion), or if the unmarshal fails. -// -// The logic is derived from and on par with: -// https://github.com/helm/helm/blob/v3.3.4/pkg/repo/index.go#L301 -func (r *ChartRepository) LoadIndex(b []byte) error { +// LoadIndexFromBytes loads Index from the given bytes. +// It returns a repo.ErrNoAPIVersion error if the API version is not set +func (r *ChartRepository) LoadIndexFromBytes(b []byte) error { i := &repo.IndexFile{} if err := yaml.UnmarshalStrict(b, i); err != nil { return err @@ -190,14 +220,68 @@ func (r *ChartRepository) LoadIndex(b []byte) error { return repo.ErrNoAPIVersion } i.SortEntries() + + r.Lock() r.Index = i + r.Checksum = fmt.Sprintf("%x", sha256.Sum256(b)) + r.Unlock() return nil } +// LoadFromFile reads the file at the given path and loads it into Index. +func (r *ChartRepository) LoadFromFile(path string) error { + b, err := os.ReadFile(path) + if err != nil { + return err + } + return r.LoadIndexFromBytes(b) +} + +// CacheIndex attempts to write the index from the remote into a new temporary file +// using DownloadIndex, and sets CachePath. +// It returns the SHA256 checksum of the downloaded index bytes, or an error. +// The caller is expected to handle the garbage collection of CachePath, and to +// load the Index separately using LoadFromCache if required. +func (r *ChartRepository) CacheIndex() (string, error) { + f, err := os.CreateTemp("", "chart-index-*.yaml") + if err != nil { + return "", fmt.Errorf("failed to create temp file to cache index to: %w", err) + } + + h := sha256.New() + mw := io.MultiWriter(f, h) + if err = r.DownloadIndex(mw); err != nil { + f.Close() + os.RemoveAll(f.Name()) + return "", fmt.Errorf("failed to cache index to '%s': %w", f.Name(), err) + } + if err = f.Close(); err != nil { + os.RemoveAll(f.Name()) + return "", fmt.Errorf("failed to close cached index file '%s': %w", f.Name(), err) + } + + r.Lock() + r.CachePath = f.Name() + r.Unlock() + return hex.EncodeToString(h.Sum(nil)), nil +} + +// LoadFromCache attempts to load the Index from the configured CachePath. +// It returns an error if no CachePath is set, or if the load failed. +func (r *ChartRepository) LoadFromCache() error { + r.RLock() + if cachePath := r.CachePath; cachePath != "" { + r.RUnlock() + return r.LoadFromFile(cachePath) + } + r.RUnlock() + return fmt.Errorf("no cache path set") +} + // DownloadIndex attempts to download the chart repository index using -// the Client and set Options, and loads the index file into the Index. -// It returns an error on URL parsing and Client failures. -func (r *ChartRepository) DownloadIndex() error { +// the Client and set Options, and writes the index to the given io.Writer. +// It returns an url.Error if the URL failed to parse. +func (r *ChartRepository) DownloadIndex(w io.Writer) (err error) { u, err := url.Parse(r.URL) if err != nil { return err @@ -205,14 +289,36 @@ func (r *ChartRepository) DownloadIndex() error { u.RawPath = path.Join(u.RawPath, "index.yaml") u.Path = path.Join(u.Path, "index.yaml") - res, err := r.Client.Get(u.String(), r.Options...) + var res *bytes.Buffer + res, err = r.Client.Get(u.String(), r.Options...) if err != nil { return err } - b, err := io.ReadAll(res) - if err != nil { + if _, err = io.Copy(w, res); err != nil { return err } + return nil +} + +// HasIndex returns true if the Index is not nil. +func (r *ChartRepository) HasIndex() bool { + r.RLock() + defer r.RUnlock() + return r.Index != nil +} + +// HasCacheFile returns true if CachePath is not empty. +func (r *ChartRepository) HasCacheFile() bool { + r.RLock() + defer r.RUnlock() + return r.CachePath != "" +} - return r.LoadIndex(b) +// UnloadIndex sets the Index to nil. +func (r *ChartRepository) UnloadIndex() { + if r != nil { + r.Lock() + r.Index = nil + r.Unlock() + } } diff --git a/internal/helm/repository_test.go b/internal/helm/repository_test.go index c51a19d40..95ccc7b80 100644 --- a/internal/helm/repository_test.go +++ b/internal/helm/repository_test.go @@ -18,45 +18,38 @@ package helm import ( "bytes" + "crypto/sha256" + "fmt" "net/url" "os" - "reflect" - "strings" "testing" "time" + . "github.com/onsi/gomega" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/repo" ) +var now = time.Now() + const ( - testfile = "testdata/local-index.yaml" - chartmuseumtestfile = "testdata/chartmuseum-index.yaml" - unorderedtestfile = "testdata/local-index-unordered.yaml" - indexWithDuplicates = ` -apiVersion: v1 -entries: - nginx: - - urls: - - https://kubernetes-charts.storage.googleapis.com/nginx-0.2.0.tgz - name: nginx - description: string - version: 0.2.0 - home: https://github.com/something/else - digest: "sha256:1234567890abcdef" - nginx: - - urls: - - https://kubernetes-charts.storage.googleapis.com/alpine-1.0.0.tgz - - http://storage2.googleapis.com/kubernetes-charts/alpine-1.0.0.tgz - name: alpine - description: string - version: 1.0.0 - home: https://github.com/something - digest: "sha256:1234567890abcdef" -` + testFile = "testdata/local-index.yaml" + chartmuseumTestFile = "testdata/chartmuseum-index.yaml" + unorderedTestFile = "testdata/local-index-unordered.yaml" ) +// mockGetter can be used as a simple mocking getter.Getter implementation. +type mockGetter struct { + requestedURL string + response []byte +} + +func (g *mockGetter) Get(url string, _ ...getter.Option) (*bytes.Buffer, error) { + g.requestedURL = url + return bytes.NewBuffer(g.response), nil +} + func TestNewChartRepository(t *testing.T) { repositoryURL := "https://example.com" providers := getter.Providers{ @@ -68,60 +61,74 @@ func TestNewChartRepository(t *testing.T) { options := []getter.Option{getter.WithBasicAuth("username", "password")} t.Run("should construct chart repository", func(t *testing.T) { - r, err := NewChartRepository(repositoryURL, providers, options) - if err != nil { - t.Error(err) - } - if got := r.URL; got != repositoryURL { - t.Fatalf("Expecting %q repository URL, got: %q", repositoryURL, got) - } - if r.Client == nil { - t.Fatalf("Expecting client, got nil") - } - if !reflect.DeepEqual(r.Options, options) { - t.Fatalf("Client options mismatth") - } + g := NewWithT(t) + + r, err := NewChartRepository(repositoryURL, "", providers, options) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(r).ToNot(BeNil()) + g.Expect(r.URL).To(Equal(repositoryURL)) + g.Expect(r.Client).ToNot(BeNil()) + g.Expect(r.Options).To(Equal(options)) }) t.Run("should error on URL parsing failure", func(t *testing.T) { - _, err := NewChartRepository("https://ex ample.com", nil, nil) - switch err.(type) { - case *url.Error: - default: - t.Fatalf("Expecting URL error, got: %v", err) - } + g := NewWithT(t) + r, err := NewChartRepository("https://ex ample.com", "", nil, nil) + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(BeAssignableToTypeOf(&url.Error{})) + g.Expect(r).To(BeNil()) + }) t.Run("should error on unsupported scheme", func(t *testing.T) { - _, err := NewChartRepository("http://example.com", providers, nil) - if err == nil { - t.Fatalf("Expecting unsupported scheme error") - } + g := NewWithT(t) + + r, err := NewChartRepository("http://example.com", "", providers, nil) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(Equal("scheme \"http\" not supported")) + g.Expect(r).To(BeNil()) }) } func TestChartRepository_Get(t *testing.T) { - i := repo.NewIndexFile() - i.Add(&chart.Metadata{Name: "chart", Version: "0.0.1"}, "chart-0.0.1.tgz", "http://example.com/charts", "sha256:1234567890") - i.Add(&chart.Metadata{Name: "chart", Version: "0.1.0"}, "chart-0.1.0.tgz", "http://example.com/charts", "sha256:1234567890abc") - i.Add(&chart.Metadata{Name: "chart", Version: "0.1.1"}, "chart-0.1.1.tgz", "http://example.com/charts", "sha256:1234567890abc") - i.Add(&chart.Metadata{Name: "chart", Version: "0.1.5+b.min.minute"}, "chart-0.1.5+b.min.minute.tgz", "http://example.com/charts", "sha256:1234567890abc") - i.Entries["chart"][len(i.Entries["chart"])-1].Created = time.Now().Add(-time.Minute) - i.Add(&chart.Metadata{Name: "chart", Version: "0.1.5+a.min.hour"}, "chart-0.1.5+a.min.hour.tgz", "http://example.com/charts", "sha256:1234567890abc") - i.Entries["chart"][len(i.Entries["chart"])-1].Created = time.Now().Add(-time.Hour) - i.Add(&chart.Metadata{Name: "chart", Version: "0.1.5+c.now"}, "chart-0.1.5+c.now.tgz", "http://example.com/charts", "sha256:1234567890abc") - i.Add(&chart.Metadata{Name: "chart", Version: "0.2.0"}, "chart-0.2.0.tgz", "http://example.com/charts", "sha256:1234567890abc") - i.Add(&chart.Metadata{Name: "chart", Version: "1.0.0"}, "chart-1.0.0.tgz", "http://example.com/charts", "sha256:1234567890abc") - i.Add(&chart.Metadata{Name: "chart", Version: "1.1.0-rc.1"}, "chart-1.1.0-rc.1.tgz", "http://example.com/charts", "sha256:1234567890abc") - i.SortEntries() - r := &ChartRepository{Index: i} + g := NewWithT(t) + + r := newChartRepository() + r.Index = repo.NewIndexFile() + charts := []struct { + name string + version string + url string + digest string + created time.Time + }{ + {name: "chart", version: "0.0.1", url: "http://example.com/charts", digest: "sha256:1234567890"}, + {name: "chart", version: "0.1.0", url: "http://example.com/charts", digest: "sha256:1234567890abc"}, + {name: "chart", version: "0.1.1", url: "http://example.com/charts", digest: "sha256:1234567890abc"}, + {name: "chart", version: "0.1.5+b.min.minute", url: "http://example.com/charts", digest: "sha256:1234567890abc", created: now.Add(-time.Minute)}, + {name: "chart", version: "0.1.5+a.min.hour", url: "http://example.com/charts", digest: "sha256:1234567890abc", created: now.Add(-time.Hour)}, + {name: "chart", version: "0.1.5+c.now", url: "http://example.com/charts", digest: "sha256:1234567890abc", created: now}, + {name: "chart", version: "0.2.0", url: "http://example.com/charts", digest: "sha256:1234567890abc"}, + {name: "chart", version: "1.0.0", url: "http://example.com/charts", digest: "sha256:1234567890abc"}, + {name: "chart", version: "1.1.0-rc.1", url: "http://example.com/charts", digest: "sha256:1234567890abc"}, + } + for _, c := range charts { + g.Expect(r.Index.MustAdd( + &chart.Metadata{Name: c.name, Version: c.version}, + fmt.Sprintf("%s-%s.tgz", c.name, c.version), c.url, c.digest), + ).To(Succeed()) + if !c.created.IsZero() { + r.Index.Entries["chart"][len(r.Index.Entries["chart"])-1].Created = c.created + } + } + r.Index.SortEntries() tests := []struct { name string chartName string chartVersion string wantVersion string - wantErr bool + wantErr string }{ { name: "exact match", @@ -151,12 +158,12 @@ func TestChartRepository_Get(t *testing.T) { name: "unfulfilled range", chartName: "chart", chartVersion: ">2.0.0", - wantErr: true, + wantErr: "no 'chart' chart with version matching '>2.0.0' found", }, { name: "invalid chart", chartName: "non-existing", - wantErr: true, + wantErr: repo.ErrNoChartName.Error(), }, { name: "match newest if ambiguous", @@ -168,14 +175,19 @@ func TestChartRepository_Get(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + cv, err := r.Get(tt.chartName, tt.chartVersion) - if (err != nil) != tt.wantErr { - t.Errorf("Get() error = %v, wantErr %v", err, tt.wantErr) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(cv).To(BeNil()) return } - if err == nil && !strings.Contains(cv.Metadata.Version, tt.wantVersion) { - t.Errorf("Get() unexpected version = %s, want = %s", cv.Metadata.Version, tt.wantVersion) - } + g.Expect(cv).ToNot(BeNil()) + g.Expect(cv.Metadata.Name).To(Equal(tt.chartName)) + g.Expect(cv.Metadata.Version).To(Equal(tt.wantVersion)) + g.Expect(err).ToNot(HaveOccurred()) }) } } @@ -212,117 +224,257 @@ func TestChartRepository_DownloadChart(t *testing.T) { }, } for _, tt := range tests { + tt := tt t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + t.Parallel() + mg := mockGetter{} r := &ChartRepository{ URL: tt.url, Client: &mg, } - _, err := r.DownloadChart(tt.chartVersion) - if (err != nil) != tt.wantErr { - t.Errorf("DownloadChart() error = %v, wantErr %v", err, tt.wantErr) + res, err := r.DownloadChart(tt.chartVersion) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(res).To(BeNil()) return } - if err == nil && mg.requestedURL != tt.wantURL { - t.Errorf("DownloadChart() requested URL = %s, wantURL %s", mg.requestedURL, tt.wantURL) - } + g.Expect(mg.requestedURL).To(Equal(tt.wantURL)) + g.Expect(res).ToNot(BeNil()) + g.Expect(err).ToNot(HaveOccurred()) }) } } func TestChartRepository_DownloadIndex(t *testing.T) { - b, err := os.ReadFile(chartmuseumtestfile) - if err != nil { - t.Fatal(err) - } + g := NewWithT(t) + + b, err := os.ReadFile(chartmuseumTestFile) + g.Expect(err).ToNot(HaveOccurred()) + mg := mockGetter{response: b} r := &ChartRepository{ URL: "https://example.com", Client: &mg, } - if err := r.DownloadIndex(); err != nil { + + buf := bytes.NewBuffer([]byte{}) + g.Expect(r.DownloadIndex(buf)).To(Succeed()) + g.Expect(buf.Bytes()).To(Equal(b)) + g.Expect(mg.requestedURL).To(Equal(r.URL + "/index.yaml")) + g.Expect(err).To(BeNil()) +} + +func TestChartRepository_LoadIndexFromBytes(t *testing.T) { + tests := []struct { + name string + b []byte + wantName string + wantVersion string + wantDigest string + wantErr string + }{ + { + name: "index", + b: []byte(` +apiVersion: v1 +entries: + nginx: + - urls: + - https://kubernetes-charts.storage.googleapis.com/nginx-0.2.0.tgz + name: nginx + description: string + version: 0.2.0 + home: https://github.com/something/else + digest: "sha256:1234567890abcdef" +`), + wantName: "nginx", + wantVersion: "0.2.0", + wantDigest: "sha256:1234567890abcdef", + }, + { + name: "index without API version", + b: []byte(`entries: + nginx: + - name: nginx`), + wantErr: "no API version specified", + }, + { + name: "index with duplicate entry", + b: []byte(`apiVersion: v1 +entries: + nginx: + - name: nginx" + nginx: + - name: nginx`), + wantErr: "key \"nginx\" already set in map", + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + t.Parallel() + + r := newChartRepository() + err := r.LoadIndexFromBytes(tt.b) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(r.Index).To(BeNil()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(r.Index).ToNot(BeNil()) + got, err := r.Index.Get(tt.wantName, tt.wantVersion) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got.Digest).To(Equal(tt.wantDigest)) + }) + } +} + +func TestChartRepository_LoadIndexFromBytes_Unordered(t *testing.T) { + b, err := os.ReadFile(unorderedTestFile) + if err != nil { t.Fatal(err) } - if expected := r.URL + "/index.yaml"; mg.requestedURL != expected { - t.Errorf("DownloadIndex() requested URL = %s, wantURL %s", mg.requestedURL, expected) + r := newChartRepository() + err = r.LoadIndexFromBytes(b) + if err != nil { + t.Fatal(err) } verifyLocalIndex(t, r.Index) } // Index load tests are derived from https://github.com/helm/helm/blob/v3.3.4/pkg/repo/index_test.go#L108 // to ensure parity with Helm behaviour. -func TestChartRepository_LoadIndex(t *testing.T) { +func TestChartRepository_LoadIndexFromFile(t *testing.T) { tests := []struct { name string filename string }{ { name: "regular index file", - filename: testfile, + filename: testFile, }, { name: "chartmuseum index file", - filename: chartmuseumtestfile, + filename: chartmuseumTestFile, }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) t.Parallel() - b, err := os.ReadFile(tt.filename) - if err != nil { - t.Fatal(err) - } - r := &ChartRepository{} - err = r.LoadIndex(b) - if err != nil { - t.Fatal(err) - } + + r := newChartRepository() + err := r.LoadFromFile(testFile) + g.Expect(err).ToNot(HaveOccurred()) + verifyLocalIndex(t, r.Index) }) } } -func TestChartRepository_LoadIndex_Duplicates(t *testing.T) { - r := &ChartRepository{} - if err := r.LoadIndex([]byte(indexWithDuplicates)); err == nil { - t.Errorf("Expected an error when duplicate entries are present") - } +func TestChartRepository_CacheIndex(t *testing.T) { + g := NewWithT(t) + + mg := mockGetter{response: []byte("foo")} + expectSum := fmt.Sprintf("%x", sha256.Sum256(mg.response)) + + r := newChartRepository() + r.URL = "https://example.com" + r.Client = &mg + + sum, err := r.CacheIndex() + g.Expect(err).To(Not(HaveOccurred())) + + g.Expect(r.CachePath).ToNot(BeEmpty()) + defer os.RemoveAll(r.CachePath) + g.Expect(r.CachePath).To(BeARegularFile()) + b, _ := os.ReadFile(r.CachePath) + + g.Expect(b).To(Equal(mg.response)) + g.Expect(sum).To(BeEquivalentTo(expectSum)) } -func TestChartRepository_LoadIndex_Unordered(t *testing.T) { - b, err := os.ReadFile(unorderedtestfile) - if err != nil { - t.Fatal(err) +func TestChartRepository_LoadIndexFromCache(t *testing.T) { + tests := []struct { + name string + cachePath string + wantErr string + }{ + { + name: "cache path", + cachePath: chartmuseumTestFile, + }, + { + name: "invalid cache path", + cachePath: "invalid", + wantErr: "open invalid: no such file", + }, + { + name: "no cache path", + cachePath: "", + wantErr: "no cache path set", + }, } - r := &ChartRepository{} - err = r.LoadIndex(b) - if err != nil { - t.Fatal(err) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + r := newChartRepository() + r.CachePath = tt.cachePath + err := r.LoadFromCache() + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(r.Index).To(BeNil()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + verifyLocalIndex(t, r.Index) + }) } - verifyLocalIndex(t, r.Index) +} + +func TestChartRepository_HasIndex(t *testing.T) { + g := NewWithT(t) + + r := newChartRepository() + g.Expect(r.HasIndex()).To(BeFalse()) + r.Index = repo.NewIndexFile() + g.Expect(r.HasIndex()).To(BeTrue()) +} + +func TestChartRepository_UnloadIndex(t *testing.T) { + g := NewWithT(t) + + r := newChartRepository() + g.Expect(r.HasIndex()).To(BeFalse()) + r.Index = repo.NewIndexFile() + r.UnloadIndex() + g.Expect(r.Index).To(BeNil()) } func verifyLocalIndex(t *testing.T, i *repo.IndexFile) { - numEntries := len(i.Entries) - if numEntries != 3 { - t.Errorf("Expected 3 entries in index file but got %d", numEntries) - } + g := NewWithT(t) - alpine, ok := i.Entries["alpine"] - if !ok { - t.Fatalf("'alpine' section not found.") - } + g.Expect(i.Entries).ToNot(BeNil()) + g.Expect(i.Entries).To(HaveLen(3), "expected 3 entries in index file") - if l := len(alpine); l != 1 { - t.Fatalf("'alpine' should have 1 chart, got %d", l) - } + alpine, ok := i.Entries["alpine"] + g.Expect(ok).To(BeTrue(), "expected 'alpine' entry to exist") + g.Expect(alpine).To(HaveLen(1), "'alpine' should have 1 entry") nginx, ok := i.Entries["nginx"] - if !ok || len(nginx) != 2 { - t.Fatalf("Expected 2 nginx entries") - } + g.Expect(ok).To(BeTrue(), "expected 'nginx' entry to exist") + g.Expect(nginx).To(HaveLen(2), "'nginx' should have 2 entries") expects := []*repo.ChartVersion{ { @@ -370,41 +522,12 @@ func verifyLocalIndex(t *testing.T, i *repo.IndexFile) { for i, tt := range tests { expect := expects[i] - if tt.Name != expect.Name { - t.Errorf("Expected name %q, got %q", expect.Name, tt.Name) - } - if tt.Description != expect.Description { - t.Errorf("Expected description %q, got %q", expect.Description, tt.Description) - } - if tt.Version != expect.Version { - t.Errorf("Expected version %q, got %q", expect.Version, tt.Version) - } - if tt.Digest != expect.Digest { - t.Errorf("Expected digest %q, got %q", expect.Digest, tt.Digest) - } - if tt.Home != expect.Home { - t.Errorf("Expected home %q, got %q", expect.Home, tt.Home) - } - - for i, url := range tt.URLs { - if url != expect.URLs[i] { - t.Errorf("Expected URL %q, got %q", expect.URLs[i], url) - } - } - for i, kw := range tt.Keywords { - if kw != expect.Keywords[i] { - t.Errorf("Expected keywords %q, got %q", expect.Keywords[i], kw) - } - } + g.Expect(tt.Name).To(Equal(expect.Name)) + g.Expect(tt.Description).To(Equal(expect.Description)) + g.Expect(tt.Version).To(Equal(expect.Version)) + g.Expect(tt.Digest).To(Equal(expect.Digest)) + g.Expect(tt.Home).To(Equal(expect.Home)) + g.Expect(tt.URLs).To(ContainElements(expect.URLs)) + g.Expect(tt.Keywords).To(ContainElements(expect.Keywords)) } } - -type mockGetter struct { - requestedURL string - response []byte -} - -func (g *mockGetter) Get(url string, options ...getter.Option) (*bytes.Buffer, error) { - g.requestedURL = url - return bytes.NewBuffer(g.response), nil -} diff --git a/internal/helm/utils_test.go b/internal/helm/utils_test.go new file mode 100644 index 000000000..62a9e92c2 --- /dev/null +++ b/internal/helm/utils_test.go @@ -0,0 +1,60 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helm + +import ( + "testing" + + . "github.com/onsi/gomega" +) + +func TestNormalizeChartRepositoryURL(t *testing.T) { + tests := []struct { + name string + url string + want string + }{ + { + name: "with slash", + url: "http://example.com/", + want: "http://example.com/", + }, + { + name: "without slash", + url: "http://example.com", + want: "http://example.com/", + }, + { + name: "double slash", + url: "http://example.com//", + want: "http://example.com/", + }, + { + name: "empty", + url: "", + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + got := NormalizeChartRepositoryURL(tt.url) + g.Expect(got).To(Equal(tt.want)) + }) + } +} From d60131d16b279e398257df27112b9a6b351fcc01 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Mon, 1 Nov 2021 09:20:48 +0100 Subject: [PATCH 03/23] internal/helm: optimize dependency manager This commit starts with the optimization of the `DepenendencyManager`, ensuring the chart indexes are lazy loaded, and replacing the (limitless) concurrency with a configurable number of workers with a default of 1. Signed-off-by: Hidde Beydals --- internal/helm/dependency_manager.go | 68 ++++++++++++++++-------- internal/helm/dependency_manager_test.go | 11 ++-- 2 files changed, 51 insertions(+), 28 deletions(-) diff --git a/internal/helm/dependency_manager.go b/internal/helm/dependency_manager.go index 83b42d4d7..19d56c884 100644 --- a/internal/helm/dependency_manager.go +++ b/internal/helm/dependency_manager.go @@ -28,6 +28,7 @@ import ( "github.com/Masterminds/semver/v3" securejoin "github.com/cyphar/filepath-securejoin" "golang.org/x/sync/errgroup" + "golang.org/x/sync/semaphore" helmchart "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" ) @@ -58,38 +59,51 @@ type DependencyManager struct { // Dependencies contains a list of dependencies, and the respective // repository the dependency can be found at. Dependencies []*DependencyWithRepository + // Workers is the number of concurrent chart-add operations during + // Build. Defaults to 1 (non-concurrent). + Workers int64 mu sync.Mutex } -// Build compiles and builds the dependencies of the Chart. +// Build compiles and builds the dependencies of the Chart with the +// configured number of Workers. func (dm *DependencyManager) Build(ctx context.Context) error { if len(dm.Dependencies) == 0 { return nil } - errs, ctx := errgroup.WithContext(ctx) - for _, i := range dm.Dependencies { - item := i - errs.Go(func() error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - } + workers := dm.Workers + if workers <= 0 { + workers = 1 + } - var err error - switch item.Repository { - case nil: - err = dm.addLocalDependency(item) - default: - err = dm.addRemoteDependency(item) + defer func() { + for _, dep := range dm.Dependencies { + dep.Repository.UnloadIndex() + } + }() + + group, groupCtx := errgroup.WithContext(ctx) + group.Go(func() error { + sem := semaphore.NewWeighted(workers) + for _, dep := range dm.Dependencies { + dep := dep + if err := sem.Acquire(groupCtx, 1); err != nil { + return err } - return err - }) - } + group.Go(func() error { + defer sem.Release(1) + if dep.Repository == nil { + return dm.addLocalDependency(dep) + } + return dm.addRemoteDependency(dep) + }) + } + return nil + }) - return errs.Wait() + return group.Wait() } func (dm *DependencyManager) addLocalDependency(dpr *DependencyWithRepository) error { @@ -136,7 +150,18 @@ func (dm *DependencyManager) addLocalDependency(dpr *DependencyWithRepository) e func (dm *DependencyManager) addRemoteDependency(dpr *DependencyWithRepository) error { if dpr.Repository == nil { - return fmt.Errorf("no ChartRepository given for '%s' dependency", dpr.Dependency.Name) + return fmt.Errorf("no HelmRepository for '%s' dependency", dpr.Dependency.Name) + } + + if !dpr.Repository.HasIndex() { + if !dpr.Repository.HasCacheFile() { + if _, err := dpr.Repository.CacheIndex(); err != nil { + return err + } + } + if err := dpr.Repository.LoadFromCache(); err != nil { + return err + } } chartVer, err := dpr.Repository.Get(dpr.Dependency.Name, dpr.Dependency.Version) @@ -157,7 +182,6 @@ func (dm *DependencyManager) addRemoteDependency(dpr *DependencyWithRepository) dm.mu.Lock() dm.Chart.AddDependency(ch) dm.mu.Unlock() - return nil } diff --git a/internal/helm/dependency_manager_test.go b/internal/helm/dependency_manager_test.go index 6a38997b2..a8e6a0480 100644 --- a/internal/helm/dependency_manager_test.go +++ b/internal/helm/dependency_manager_test.go @@ -182,13 +182,12 @@ func TestBuild_WithRemoteChart(t *testing.T) { t.Fatal(err) } i := repo.NewIndexFile() - i.Add(&helmchart.Metadata{Name: chartName, Version: chartVersion}, fmt.Sprintf("%s-%s.tgz", chartName, chartVersion), "http://example.com/charts", "sha256:1234567890") + i.MustAdd(&helmchart.Metadata{Name: chartName, Version: chartVersion}, fmt.Sprintf("%s-%s.tgz", chartName, chartVersion), "http://example.com/charts", "sha256:1234567890") mg := mockGetter{response: b} - cr := &ChartRepository{ - URL: remoteDepFixture.Repository, - Index: i, - Client: &mg, - } + cr := newChartRepository() + cr.URL = remoteDepFixture.Repository + cr.Index = i + cr.Client = &mg dm := DependencyManager{ Chart: &chart, Dependencies: []*DependencyWithRepository{ From f5f212ff430391c579b78f1ee56db2ec1be54166 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 4 Nov 2021 17:31:22 +0100 Subject: [PATCH 04/23] internal/helm: introduce ChartBuilder This commit starts with the creation of a `ChartBuilder` to facilitate the (conditional) build of a chart outside of the reconciler logic. The builder can be configured with a set of (modifying) options, which define together with the type of chart source what steps are taken during the build. To better facilitate the builder's needs and attempt to be more efficient, changes have been made to the `DependencyBuilder` and `ChartRepository` around (order of) operations and/or lazy-load capabilities. Signed-off-by: Hidde Beydals --- internal/helm/chart.go | 12 +- internal/helm/chart_builder.go | 384 +++++++++++ internal/helm/chart_builder_test.go | 598 +++++++++++++++++ internal/helm/chart_test.go | 35 +- internal/helm/dependency_manager.go | 265 ++++++-- internal/helm/dependency_manager_test.go | 634 +++++++++++++++--- internal/helm/repository.go | 70 +- internal/helm/repository_test.go | 7 +- .../charts/helmchart/values-prod.yaml | 1 + .../charts/helmchartwithdeps/Chart.lock | 12 + 10 files changed, 1796 insertions(+), 222 deletions(-) create mode 100644 internal/helm/chart_builder.go create mode 100644 internal/helm/chart_builder_test.go create mode 100644 internal/helm/testdata/charts/helmchart/values-prod.yaml create mode 100644 internal/helm/testdata/charts/helmchartwithdeps/Chart.lock diff --git a/internal/helm/chart.go b/internal/helm/chart.go index accbc69a9..dcc868c1d 100644 --- a/internal/helm/chart.go +++ b/internal/helm/chart.go @@ -70,17 +70,17 @@ func OverwriteChartDefaultValues(chart *helmchart.Chart, data []byte) (bool, err // LoadChartMetadata attempts to load the chart.Metadata from the "Chart.yaml" file in the directory or archive at the // given chartPath. It takes "requirements.yaml" files into account, and is therefore compatible with the // chart.APIVersionV1 format. -func LoadChartMetadata(chartPath string) (*helmchart.Metadata, error) { +func LoadChartMetadata(chartPath string) (meta *helmchart.Metadata, err error) { i, err := os.Stat(chartPath) if err != nil { return nil, err } - switch { - case i.IsDir(): - return LoadChartMetadataFromDir(chartPath) - default: - return LoadChartMetadataFromArchive(chartPath) + if i.IsDir() { + meta, err = LoadChartMetadataFromDir(chartPath) + return } + meta, err = LoadChartMetadataFromArchive(chartPath) + return } // LoadChartMetadataFromDir loads the chart.Metadata from the "Chart.yaml" file in the directory at the given path. diff --git a/internal/helm/chart_builder.go b/internal/helm/chart_builder.go new file mode 100644 index 000000000..7b90cba81 --- /dev/null +++ b/internal/helm/chart_builder.go @@ -0,0 +1,384 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helm + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + securejoin "github.com/cyphar/filepath-securejoin" + "github.com/fluxcd/source-controller/internal/fs" + helmchart "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/chartutil" + "sigs.k8s.io/yaml" + + "github.com/fluxcd/pkg/runtime/transform" +) + +// ChartBuilder aims to efficiently build a Helm chart from a directory or packaged chart. +// It avoids or delays loading the chart into memory in full, working with chart.Metadata +// as much as it can, and returns early (by copying over the already packaged source chart) +// if no modifications were made during the build process. +type ChartBuilder struct { + // baseDir is the chroot for the chart builder when path isDir. + // It must be (a higher) relative to path. File references (during e.g. + // value file merge operations) are not allowed to traverse out of it. + baseDir string + + // path is the file or directory path to a chart source. + path string + + // chart holds a (partly) loaded chart.Chart, it contains at least the + // chart.Metadata, which may expand to the full chart.Chart if required + // for Build operations. + chart *helmchart.Chart + + // valueFiles holds a list of path references of valueFiles that should be + // merged and packaged as a single "values.yaml" during Build. + valueFiles []string + + // repositories holds an index of repository URLs and their ChartRepository. + // They are used to configure a DependencyManager for missing chart dependencies + // if isDir is true. + repositories map[string]*ChartRepository + + // getChartRepositoryCallback is used to configure a DependencyManager for + // missing chart dependencies if isDir is true. + getChartRepositoryCallback GetChartRepositoryCallback + + mu sync.Mutex +} + +// NewChartBuilder constructs a new ChartBuilder for the given chart path. +// It returns an error if no chart.Metadata can be loaded from the path. +func NewChartBuilder(path string) (*ChartBuilder, error) { + metadata, err := LoadChartMetadata(path) + if err != nil { + return nil, fmt.Errorf("could not create new chart builder: %w", err) + } + return &ChartBuilder{ + path: path, + chart: &helmchart.Chart{ + Metadata: metadata, + }, + }, nil +} + +// WithBaseDir configures the base dir on the ChartBuilder. +func (b *ChartBuilder) WithBaseDir(p string) *ChartBuilder { + b.mu.Lock() + b.baseDir = p + b.mu.Unlock() + return b +} + +// WithValueFiles appends the given paths to the ChartBuilder's valueFiles. +func (b *ChartBuilder) WithValueFiles(path ...string) *ChartBuilder { + b.mu.Lock() + b.valueFiles = append(b.valueFiles, path...) + b.mu.Unlock() + return b +} + +// WithChartRepository indexes the given ChartRepository by the NormalizeChartRepositoryURL, +// used to configure the DependencyManager if the chart is not packaged. +func (b *ChartBuilder) WithChartRepository(url string, index *ChartRepository) *ChartBuilder { + b.mu.Lock() + b.repositories[NormalizeChartRepositoryURL(url)] = index + b.mu.Unlock() + return b +} + +// WithChartRepositoryCallback configures the GetChartRepositoryCallback used by the +// DependencyManager if the chart is not packaged. +func (b *ChartBuilder) WithChartRepositoryCallback(c GetChartRepositoryCallback) *ChartBuilder { + b.mu.Lock() + b.getChartRepositoryCallback = c + b.mu.Unlock() + return b +} + +// ChartBuildResult contains the ChartBuilder result, including build specific +// information about the chart. +type ChartBuildResult struct { + // SourceIsDir indicates if the chart was build from a directory. + SourceIsDir bool + // Path contains the absolute path to the packaged chart. + Path string + // ValuesOverwrite holds a structured map with the merged values used + // to overwrite chart default "values.yaml". + ValuesOverwrite map[string]interface{} + // CollectedDependencies contains the number of missing local and remote + // dependencies that were collected by the DependencyManager before building + // the chart. + CollectedDependencies int + // Packaged indicates if the ChartBuilder has packaged the chart. + // This can for example be false if SourceIsDir is false and ValuesOverwrite + // is nil, which makes the ChartBuilder copy the chart source to Path without + // making any modifications. + Packaged bool +} + +// String returns the Path of the ChartBuildResult. +func (b *ChartBuildResult) String() string { + if b != nil { + return b.Path + } + return "" +} + +// Build attempts to build a new chart using ChartBuilder configuration, +// writing it to the provided path. +// It returns a ChartBuildResult containing all information about the resulting chart, +// or an error. +func (b *ChartBuilder) Build(ctx context.Context, p string) (_ *ChartBuildResult, err error) { + b.mu.Lock() + defer b.mu.Unlock() + + if b.chart == nil { + err = fmt.Errorf("chart build failed: no initial chart (metadata) loaded") + return + } + if b.path == "" { + err = fmt.Errorf("chart build failed: no path set") + return + } + + result := &ChartBuildResult{} + result.SourceIsDir = pathIsDir(b.path) + result.Path = p + + // Merge chart values + if err = b.mergeValues(result); err != nil { + err = fmt.Errorf("chart build failed: %w", err) + return + } + + // Ensure chart has all dependencies + if err = b.buildDependencies(ctx, result); err != nil { + err = fmt.Errorf("chart build failed: %w", err) + return + } + + // Package (or copy) chart + if err = b.packageChart(result); err != nil { + err = fmt.Errorf("chart package failed: %w", err) + return + } + return result, nil +} + +// load lazy-loads chart.Chart into chart from the set path, it replaces any previously set +// chart.Metadata shim. +func (b *ChartBuilder) load() (err error) { + if b.chart == nil || len(b.chart.Files) <= 0 { + if b.path == "" { + return fmt.Errorf("failed to load chart: path not set") + } + chart, err := loader.Load(b.path) + if err != nil { + return fmt.Errorf("failed to load chart: %w", err) + } + b.chart = chart + } + return +} + +// buildDependencies builds the missing dependencies for a chart from a directory. +// Using the chart using a NewDependencyManager and the configured repositories +// and getChartRepositoryCallback +// It returns the number of dependencies it collected, or an error. +func (b *ChartBuilder) buildDependencies(ctx context.Context, result *ChartBuildResult) (err error) { + if !result.SourceIsDir { + return + } + + if err = b.load(); err != nil { + err = fmt.Errorf("failed to ensure chart has no missing dependencies: %w", err) + return + } + + dm := NewDependencyManager(b.chart, b.baseDir, strings.TrimLeft(b.path, b.baseDir)). + WithRepositories(b.repositories). + WithChartRepositoryCallback(b.getChartRepositoryCallback) + + result.CollectedDependencies, err = dm.Build(ctx) + return +} + +// mergeValues strategically merges the valueFiles, it merges using mergeFileValues +// or mergeChartValues depending on if the chart is sourced from a package or directory. +// Ir only calls load to propagate the chart if required by the strategy. +// It returns the merged values, or an error. +func (b *ChartBuilder) mergeValues(result *ChartBuildResult) (err error) { + if len(b.valueFiles) == 0 { + return + } + + if result.SourceIsDir { + result.ValuesOverwrite, err = mergeFileValues(b.baseDir, b.valueFiles) + if err != nil { + err = fmt.Errorf("failed to merge value files: %w", err) + } + return + } + + // Values equal to default + if len(b.valueFiles) == 1 && b.valueFiles[0] == chartutil.ValuesfileName { + return + } + + if err = b.load(); err != nil { + err = fmt.Errorf("failed to merge chart values: %w", err) + return + } + + if result.ValuesOverwrite, err = mergeChartValues(b.chart, b.valueFiles); err != nil { + err = fmt.Errorf("failed to merge chart values: %w", err) + return + } + return nil +} + +// packageChart determines if it should copyFileToPath or packageToPath +// based on the provided result. It sets Packaged on ChartBuildResult to +// true if packageToPath is successful. +func (b *ChartBuilder) packageChart(result *ChartBuildResult) error { + // If we are not building from a directory, and we do not have any + // replacement values, we can copy over the already packaged source + // chart without making any modifications + if !result.SourceIsDir && len(result.ValuesOverwrite) == 0 { + if err := copyFileToPath(b.path, result.Path); err != nil { + return fmt.Errorf("chart build failed: %w", err) + } + return nil + } + + // Package chart to a new temporary directory + if err := packageToPath(b.chart, result.Path); err != nil { + return fmt.Errorf("chart build failed: %w", err) + } + result.Packaged = true + return nil +} + +// mergeChartValues merges the given chart.Chart Files paths into a single "values.yaml" map. +// It returns the merge result, or an error. +func mergeChartValues(chart *helmchart.Chart, paths []string) (map[string]interface{}, error) { + mergedValues := make(map[string]interface{}) + for _, p := range paths { + cfn := filepath.Clean(p) + if cfn == chartutil.ValuesfileName { + mergedValues = transform.MergeMaps(mergedValues, chart.Values) + continue + } + var b []byte + for _, f := range chart.Files { + if f.Name == cfn { + b = f.Data + break + } + } + if b == nil { + return nil, fmt.Errorf("no values file found at path '%s'", p) + } + values := make(map[string]interface{}) + if err := yaml.Unmarshal(b, &values); err != nil { + return nil, fmt.Errorf("unmarshaling values from '%s' failed: %w", p, err) + } + mergedValues = transform.MergeMaps(mergedValues, values) + } + return mergedValues, nil +} + +// mergeFileValues merges the given value file paths into a single "values.yaml" map. +// The provided (relative) paths may not traverse outside baseDir. It returns the merge +// result, or an error. +func mergeFileValues(baseDir string, paths []string) (map[string]interface{}, error) { + mergedValues := make(map[string]interface{}) + for _, p := range paths { + secureP, err := securejoin.SecureJoin(baseDir, p) + if err != nil { + return nil, err + } + if f, err := os.Stat(secureP); os.IsNotExist(err) || !f.Mode().IsRegular() { + return nil, fmt.Errorf("no values file found at path '%s' (reference '%s')", + strings.TrimPrefix(secureP, baseDir), p) + } + b, err := os.ReadFile(secureP) + if err != nil { + return nil, fmt.Errorf("could not read values from file '%s': %w", p, err) + } + values := make(map[string]interface{}) + err = yaml.Unmarshal(b, &values) + if err != nil { + return nil, fmt.Errorf("unmarshaling values from '%s' failed: %w", p, err) + } + mergedValues = transform.MergeMaps(mergedValues, values) + } + return mergedValues, nil +} + +// copyFileToPath attempts to copy in to out. It returns an error if out already exists. +func copyFileToPath(in, out string) error { + o, err := os.Create(out) + if err != nil { + return fmt.Errorf("failed to create copy target: %w", err) + } + defer o.Close() + i, err := os.Open(in) + if err != nil { + return fmt.Errorf("failed to open file to copy from: %w", err) + } + defer i.Close() + if _, err := o.ReadFrom(i); err != nil { + return fmt.Errorf("failed to read from source during copy: %w", err) + } + return nil +} + +// packageToPath attempts to package the given chart.Chart to the out filepath. +func packageToPath(chart *helmchart.Chart, out string) error { + o, err := os.MkdirTemp("", "chart-build-*") + if err != nil { + return fmt.Errorf("failed to create temporary directory for chart: %w", err) + } + defer os.RemoveAll(o) + + p, err := chartutil.Save(chart, o) + if err != nil { + return fmt.Errorf("failed to package chart: %w", err) + } + return fs.RenameWithFallback(p, out) +} + +// pathIsDir returns a boolean indicating if the given path points to a directory. +// In case os.Stat on the given path returns an error it returns false as well. +func pathIsDir(p string) bool { + if p == "" { + return false + } + if i, err := os.Stat(p); err != nil || !i.IsDir() { + return false + } + return true +} diff --git a/internal/helm/chart_builder_test.go b/internal/helm/chart_builder_test.go new file mode 100644 index 000000000..afc0107ce --- /dev/null +++ b/internal/helm/chart_builder_test.go @@ -0,0 +1,598 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helm + +import ( + "context" + "encoding/hex" + "fmt" + "math/rand" + "os" + "path/filepath" + "sync" + "testing" + + . "github.com/onsi/gomega" + helmchart "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/chartutil" + "helm.sh/helm/v3/pkg/repo" +) + +func TestChartBuildResult_String(t *testing.T) { + g := NewWithT(t) + + var result *ChartBuildResult + g.Expect(result.String()).To(Equal("")) + result = &ChartBuildResult{} + g.Expect(result.String()).To(Equal("")) + result = &ChartBuildResult{Path: "/foo/"} + g.Expect(result.String()).To(Equal("/foo/")) +} + +func TestChartBuilder_Build(t *testing.T) { + tests := []struct { + name string + baseDir string + path string + valueFiles []string + repositories map[string]*ChartRepository + getChartRepositoryCallback GetChartRepositoryCallback + wantErr string + }{ + { + name: "builds chart from directory", + path: "testdata/charts/helmchart", + }, + { + name: "builds chart from package", + path: "testdata/charts/helmchart-0.1.0.tgz", + }, + { + // TODO(hidde): add more diverse tests + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + b, err := NewChartBuilder(tt.path) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(b).ToNot(BeNil()) + + b.WithBaseDir(tt.baseDir) + b.WithValueFiles(tt.valueFiles...) + b.WithChartRepositoryCallback(b.getChartRepositoryCallback) + for k, v := range tt.repositories { + b.WithChartRepository(k, v) + } + + out := tmpFile("build-0.1.0", ".tgz") + defer os.RemoveAll(out) + got, err := b.Build(context.TODO(), out) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(got).To(BeNil()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).ToNot(BeNil()) + + g.Expect(got.Path).ToNot(BeEmpty()) + g.Expect(got.Path).To(Equal(out)) + g.Expect(got.Path).To(BeARegularFile()) + _, err = loader.Load(got.Path) + g.Expect(err).ToNot(HaveOccurred()) + }) + } +} + +func TestChartBuilder_load(t *testing.T) { + tests := []struct { + name string + path string + chart *helmchart.Chart + wantFunc func(g *WithT, c *helmchart.Chart) + wantErr string + }{ + { + name: "loads chart", + chart: nil, + path: "testdata/charts/helmchart-0.1.0.tgz", + wantFunc: func(g *WithT, c *helmchart.Chart) { + g.Expect(c.Metadata.Name).To(Equal("helmchart")) + g.Expect(c.Files).ToNot(BeZero()) + }, + }, + { + name: "overwrites chart without any files (metadata shim)", + chart: &helmchart.Chart{ + Metadata: &helmchart.Metadata{Name: "dummy"}, + }, + path: "testdata/charts/helmchart-0.1.0.tgz", + wantFunc: func(g *WithT, c *helmchart.Chart) { + g.Expect(c.Metadata.Name).To(Equal("helmchart")) + g.Expect(c.Files).ToNot(BeZero()) + }, + }, + { + name: "does not overwrite loaded chart", + chart: &helmchart.Chart{ + Metadata: &helmchart.Metadata{Name: "dummy"}, + Files: []*helmchart.File{ + {Name: "mock.yaml", Data: []byte("loaded chart")}, + }, + }, + path: "testdata/charts/helmchart-0.1.0.tgz", + wantFunc: func(g *WithT, c *helmchart.Chart) { + g.Expect(c.Metadata.Name).To(Equal("dummy")) + g.Expect(c.Files).To(HaveLen(1)) + }, + }, + { + name: "no path", + wantErr: "failed to load chart: path not set", + }, + { + name: "invalid chart", + path: "testdata/charts/empty.tgz", + wantErr: "failed to load chart: no files in chart archive", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + b := &ChartBuilder{ + path: tt.path, + chart: tt.chart, + } + err := b.load() + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + if tt.wantFunc != nil { + tt.wantFunc(g, b.chart) + } + }) + } +} + +func TestChartBuilder_buildDependencies(t *testing.T) { + g := NewWithT(t) + + chartB, err := os.ReadFile("testdata/charts/helmchart-0.1.0.tgz") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(chartB).ToNot(BeEmpty()) + + mockRepo := func() *ChartRepository { + return &ChartRepository{ + Client: &mockGetter{ + response: chartB, + }, + Index: &repo.IndexFile{ + Entries: map[string]repo.ChartVersions{ + "grafana": { + &repo.ChartVersion{ + Metadata: &helmchart.Metadata{ + Name: "grafana", + Version: "6.17.4", + }, + URLs: []string{"https://example.com/chart.tgz"}, + }, + }, + }, + }, + RWMutex: &sync.RWMutex{}, + } + } + + var mockCallback GetChartRepositoryCallback = func(url string) (*ChartRepository, error) { + if url == "https://grafana.github.io/helm-charts/" { + return mockRepo(), nil + } + return nil, fmt.Errorf("no repository for URL") + } + + tests := []struct { + name string + baseDir string + path string + chart *helmchart.Chart + fromDir bool + repositories map[string]*ChartRepository + getChartRepositoryCallback GetChartRepositoryCallback + wantCollectedDependencies int + wantErr string + }{ + { + name: "builds dependencies using callback", + fromDir: true, + baseDir: "testdata/charts", + path: "testdata/charts/helmchartwithdeps", + getChartRepositoryCallback: mockCallback, + wantCollectedDependencies: 2, + }, + { + name: "builds dependencies using repositories", + fromDir: true, + baseDir: "testdata/charts", + path: "testdata/charts/helmchartwithdeps", + repositories: map[string]*ChartRepository{ + "https://grafana.github.io/helm-charts/": mockRepo(), + }, + wantCollectedDependencies: 2, + }, + { + name: "skips dependency build for packaged chart", + path: "testdata/charts/helmchart-0.1.0.tgz", + }, + { + name: "attempts to load chart", + fromDir: true, + path: "testdata", + wantErr: "failed to ensure chart has no missing dependencies", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + b := &ChartBuilder{ + baseDir: tt.baseDir, + path: tt.path, + chart: tt.chart, + repositories: tt.repositories, + getChartRepositoryCallback: tt.getChartRepositoryCallback, + } + + result := &ChartBuildResult{SourceIsDir: tt.fromDir} + err := b.buildDependencies(context.TODO(), result) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(result.CollectedDependencies).To(BeZero()) + g.Expect(b.chart).To(Equal(tt.chart)) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(result).ToNot(BeNil()) + g.Expect(result.CollectedDependencies).To(Equal(tt.wantCollectedDependencies)) + if tt.wantCollectedDependencies > 0 { + g.Expect(b.chart).ToNot(Equal(tt.chart)) + } + }) + } +} + +func TestChartBuilder_mergeValues(t *testing.T) { + tests := []struct { + name string + baseDir string + path string + isDir bool + chart *helmchart.Chart + valueFiles []string + want map[string]interface{} + wantErr string + }{ + { + name: "merges chart values", + chart: &helmchart.Chart{ + Files: []*helmchart.File{ + {Name: "a.yaml", Data: []byte("a: b")}, + {Name: "b.yaml", Data: []byte("a: c")}, + }, + }, + valueFiles: []string{"a.yaml", "b.yaml"}, + want: map[string]interface{}{ + "a": "c", + }, + }, + { + name: "chart values merge error", + chart: &helmchart.Chart{ + Files: []*helmchart.File{ + {Name: "b.yaml", Data: []byte("a: c")}, + }, + }, + valueFiles: []string{"a.yaml"}, + wantErr: "failed to merge chart values", + }, + { + name: "merges file values", + isDir: true, + baseDir: "testdata/charts", + path: "helmchart", + valueFiles: []string{"helmchart/values-prod.yaml"}, + want: map[string]interface{}{ + "replicaCount": float64(2), + }, + }, + { + name: "file values merge error", + isDir: true, + baseDir: "testdata/charts", + path: "helmchart", + valueFiles: []string{"invalid.yaml"}, + wantErr: "failed to merge value files", + }, + { + name: "error on chart load failure", + baseDir: "testdata/charts", + path: "invalid", + wantErr: "failed to load chart", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + b := &ChartBuilder{ + baseDir: tt.baseDir, + path: tt.path, + chart: tt.chart, + valueFiles: tt.valueFiles, + } + + result := &ChartBuildResult{SourceIsDir: tt.isDir} + err := b.mergeValues(result) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(result.ValuesOverwrite).To(BeNil()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(result.ValuesOverwrite).To(Equal(tt.want)) + }) + } +} + +func Test_mergeChartValues(t *testing.T) { + tests := []struct { + name string + chart *helmchart.Chart + paths []string + want map[string]interface{} + wantErr string + }{ + { + name: "merges values", + chart: &helmchart.Chart{ + Files: []*helmchart.File{ + {Name: "a.yaml", Data: []byte("a: b")}, + {Name: "b.yaml", Data: []byte("b: c")}, + {Name: "c.yaml", Data: []byte("b: d")}, + }, + }, + paths: []string{"a.yaml", "b.yaml", "c.yaml"}, + want: map[string]interface{}{ + "a": "b", + "b": "d", + }, + }, + { + name: "uses chart values", + chart: &helmchart.Chart{ + Files: []*helmchart.File{ + {Name: "c.yaml", Data: []byte("b: d")}, + }, + Values: map[string]interface{}{ + "a": "b", + }, + }, + paths: []string{chartutil.ValuesfileName, "c.yaml"}, + want: map[string]interface{}{ + "a": "b", + "b": "d", + }, + }, + { + name: "unmarshal error", + chart: &helmchart.Chart{ + Files: []*helmchart.File{ + {Name: "invalid", Data: []byte("abcd")}, + }, + }, + paths: []string{"invalid"}, + wantErr: "unmarshaling values from 'invalid' failed", + }, + { + name: "error on invalid path", + chart: &helmchart.Chart{}, + paths: []string{"a.yaml"}, + wantErr: "no values file found at path 'a.yaml'", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + got, err := mergeChartValues(tt.chart, tt.paths) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(got).To(BeNil()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(Equal(tt.want)) + }) + } +} + +func Test_mergeFileValues(t *testing.T) { + tests := []struct { + name string + files []*helmchart.File + paths []string + want map[string]interface{} + wantErr string + }{ + { + name: "merges values from files", + files: []*helmchart.File{ + {Name: "a.yaml", Data: []byte("a: b")}, + {Name: "b.yaml", Data: []byte("b: c")}, + {Name: "c.yaml", Data: []byte("b: d")}, + }, + paths: []string{"a.yaml", "b.yaml", "c.yaml"}, + want: map[string]interface{}{ + "a": "b", + "b": "d", + }, + }, + { + name: "illegal traverse", + paths: []string{"../../../traversing/illegally/a/p/a/b"}, + wantErr: "no values file found at path '/traversing/illegally/a/p/a/b'", + }, + { + name: "unmarshal error", + files: []*helmchart.File{ + {Name: "invalid", Data: []byte("abcd")}, + }, + paths: []string{"invalid"}, + wantErr: "unmarshaling values from 'invalid' failed", + }, + { + name: "error on invalid path", + paths: []string{"a.yaml"}, + wantErr: "no values file found at path '/a.yaml'", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + baseDir, err := os.MkdirTemp("", "merge-file-values-*") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(baseDir) + + for _, f := range tt.files { + g.Expect(os.WriteFile(filepath.Join(baseDir, f.Name), f.Data, 0644)).To(Succeed()) + } + + got, err := mergeFileValues(baseDir, tt.paths) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(got).To(BeNil()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(Equal(tt.want)) + }) + } +} + +func Test_copyFileToPath(t *testing.T) { + tests := []struct { + name string + in string + wantErr string + }{ + { + name: "copies input file", + in: "testdata/local-index.yaml", + }, + { + name: "invalid input file", + in: "testdata/invalid.tgz", + wantErr: "failed to open file to copy from", + }, + { + name: "invalid input directory", + in: "testdata/charts", + wantErr: "failed to read from source during copy", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + out := tmpFile("copy-0.1.0", ".tgz") + defer os.RemoveAll(out) + err := copyFileToPath(tt.in, out) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(out).To(BeARegularFile()) + f1, err := os.ReadFile(tt.in) + g.Expect(err).ToNot(HaveOccurred()) + f2, err := os.ReadFile(out) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(f2).To(Equal(f1)) + }) + } +} + +func Test_packageToPath(t *testing.T) { + g := NewWithT(t) + + chart, err := loader.Load("testdata/charts/helmchart-0.1.0.tgz") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(chart).ToNot(BeNil()) + + out := tmpFile("chart-0.1.0", ".tgz") + defer os.RemoveAll(out) + err = packageToPath(chart, out) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(out).To(BeARegularFile()) + _, err = loader.Load(out) + g.Expect(err).ToNot(HaveOccurred()) +} + +func Test_pathIsDir(t *testing.T) { + tests := []struct { + name string + p string + want bool + }{ + {name: "directory", p: "testdata/", want: true}, + {name: "file", p: "testdata/local-index.yaml", want: false}, + {name: "not found error", p: "testdata/does-not-exist.yaml", want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + g.Expect(pathIsDir(tt.p)).To(Equal(tt.want)) + }) + } +} + +func tmpFile(prefix, suffix string) string { + randBytes := make([]byte, 16) + rand.Read(randBytes) + return filepath.Join(os.TempDir(), prefix+hex.EncodeToString(randBytes)+suffix) +} diff --git a/internal/helm/chart_test.go b/internal/helm/chart_test.go index 7afa2a3f6..23d50b96b 100644 --- a/internal/helm/chart_test.go +++ b/internal/helm/chart_test.go @@ -17,7 +17,6 @@ limitations under the License. package helm import ( - "reflect" "testing" . "github.com/onsi/gomega" @@ -87,33 +86,35 @@ func TestOverwriteChartDefaultValues(t *testing.T) { } for _, tt := range testCases { t.Run(tt.desc, func(t *testing.T) { + g := NewWithT(t) + fixture := tt.chart ok, err := OverwriteChartDefaultValues(&fixture, tt.data) - if ok != tt.ok { - t.Fatalf("should return %v, returned %v", tt.ok, ok) - } - if err != nil && !tt.expectErr { - t.Fatalf("returned unexpected error: %v", err) - } - if err == nil && tt.expectErr { - t.Fatal("expected error") + g.Expect(ok).To(Equal(tt.ok)) + + if tt.expectErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(ok).To(Equal(tt.ok)) + return } - for _, f := range fixture.Raw { - if f.Name == chartutil.ValuesfileName && reflect.DeepEqual(f.Data, originalValuesFixture) && tt.ok { - t.Error("should override values.yaml in Raw field") + if tt.ok { + for _, f := range fixture.Raw { + if f.Name == chartutil.ValuesfileName { + g.Expect(f.Data).To(Equal(tt.data)) + } } - } - for _, f := range fixture.Files { - if f.Name == chartutil.ValuesfileName && reflect.DeepEqual(f.Data, originalValuesFixture) && tt.ok { - t.Error("should override values.yaml in Files field") + for _, f := range fixture.Files { + if f.Name == chartutil.ValuesfileName { + g.Expect(f.Data).To(Equal(tt.data)) + } } } }) } } -func Test_LoadChartMetadataFromDir(t *testing.T) { +func TestLoadChartMetadataFromDir(t *testing.T) { tests := []struct { name string dir string diff --git a/internal/helm/dependency_manager.go b/internal/helm/dependency_manager.go index 19d56c884..043b0e7e3 100644 --- a/internal/helm/dependency_manager.go +++ b/internal/helm/dependency_manager.go @@ -33,165 +33,282 @@ import ( "helm.sh/helm/v3/pkg/chart/loader" ) -// DependencyWithRepository is a container for a Helm chart dependency -// and its respective repository. -type DependencyWithRepository struct { - // Dependency holds the reference to a chart.Chart dependency. - Dependency *helmchart.Dependency - // Repository is the ChartRepository the dependency should be - // available at and can be downloaded from. If there is none, - // a local ('file://') dependency is assumed. - Repository *ChartRepository -} +// GetChartRepositoryCallback must return a ChartRepository for the URL, +// or an error describing why it could not be returned. +type GetChartRepositoryCallback func(url string) (*ChartRepository, error) -// DependencyManager manages dependencies for a Helm chart. +// DependencyManager manages dependencies for a Helm chart, downloading +// only those that are missing from the chart it holds. type DependencyManager struct { - // WorkingDir is the chroot path for dependency manager operations, + // chart contains the chart.Chart from the path. + chart *helmchart.Chart + + // baseDir is the chroot path for dependency manager operations, // Dependencies that hold a local (relative) path reference are not // allowed to traverse outside this directory. - WorkingDir string - // ChartPath is the path of the Chart relative to the WorkingDir, - // the combination of the WorkingDir and ChartPath is used to + baseDir string + + // path is the path of the chart relative to the baseDir, + // the combination of the baseDir and path is used to // determine the absolute path of a local dependency. - ChartPath string - // Chart holds the loaded chart.Chart from the ChartPath. - Chart *helmchart.Chart - // Dependencies contains a list of dependencies, and the respective - // repository the dependency can be found at. - Dependencies []*DependencyWithRepository - // Workers is the number of concurrent chart-add operations during + path string + + // repositories contains a map of ChartRepository indexed by their + // normalized URL. It is used as a lookup table for missing + // dependencies. + repositories map[string]*ChartRepository + + // getChartRepositoryCallback can be set to an on-demand get + // callback which returned result is cached to repositories. + getChartRepositoryCallback GetChartRepositoryCallback + + // workers is the number of concurrent chart-add operations during // Build. Defaults to 1 (non-concurrent). - Workers int64 + workers int64 + // mu contains the lock for chart writes. mu sync.Mutex } -// Build compiles and builds the dependencies of the Chart with the -// configured number of Workers. -func (dm *DependencyManager) Build(ctx context.Context) error { - if len(dm.Dependencies) == 0 { - return nil +func NewDependencyManager(chart *helmchart.Chart, baseDir, path string) *DependencyManager { + return &DependencyManager{ + chart: chart, + baseDir: baseDir, + path: path, + } +} + +func (dm *DependencyManager) WithRepositories(r map[string]*ChartRepository) *DependencyManager { + dm.repositories = r + return dm +} + +func (dm *DependencyManager) WithChartRepositoryCallback(c GetChartRepositoryCallback) *DependencyManager { + dm.getChartRepositoryCallback = c + return dm +} + +func (dm *DependencyManager) WithWorkers(w int64) *DependencyManager { + dm.workers = w + return dm +} + +// Build compiles and builds the dependencies of the chart with the +// configured number of workers. +func (dm *DependencyManager) Build(ctx context.Context) (int, error) { + // Collect dependency metadata + var ( + deps = dm.chart.Dependencies() + reqs = dm.chart.Metadata.Dependencies + ) + // Lock file takes precedence + if lock := dm.chart.Lock; lock != nil { + reqs = lock.Dependencies + } + + // Collect missing dependencies + missing := collectMissing(deps, reqs) + if len(missing) == 0 { + return 0, nil + } + + // Run the build for the missing dependencies + if err := dm.build(ctx, missing); err != nil { + return 0, err } + return len(missing), nil +} - workers := dm.Workers +// build (concurrently) adds the given list of deps to the chart with the configured +// number of workers. It returns the first error, cancelling all other workers. +func (dm *DependencyManager) build(ctx context.Context, deps map[string]*helmchart.Dependency) error { + workers := dm.workers if workers <= 0 { workers = 1 } + // Garbage collect temporary cached ChartRepository indexes defer func() { - for _, dep := range dm.Dependencies { - dep.Repository.UnloadIndex() + for _, v := range dm.repositories { + v.Unload() + _ = v.RemoveCache() } }() group, groupCtx := errgroup.WithContext(ctx) group.Go(func() error { sem := semaphore.NewWeighted(workers) - for _, dep := range dm.Dependencies { - dep := dep + for name, dep := range deps { + name, dep := name, dep if err := sem.Acquire(groupCtx, 1); err != nil { return err } - group.Go(func() error { + group.Go(func() (err error) { defer sem.Release(1) - if dep.Repository == nil { - return dm.addLocalDependency(dep) + if isLocalDep(dep) { + if err = dm.addLocalDependency(dep); err != nil { + err = fmt.Errorf("failed to add local dependency '%s': %w", name, err) + } + return } - return dm.addRemoteDependency(dep) + if err = dm.addRemoteDependency(dep); err != nil { + err = fmt.Errorf("failed to add remote dependency '%s': %w", name, err) + } + return }) } return nil }) - return group.Wait() } -func (dm *DependencyManager) addLocalDependency(dpr *DependencyWithRepository) error { - sLocalChartPath, err := dm.secureLocalChartPath(dpr) +// addLocalDependency attempts to resolve and add the given local chart.Dependency to the chart. +func (dm *DependencyManager) addLocalDependency(dep *helmchart.Dependency) error { + sLocalChartPath, err := dm.secureLocalChartPath(dep) if err != nil { return err } if _, err := os.Stat(sLocalChartPath); err != nil { if os.IsNotExist(err) { - return fmt.Errorf("no chart found at '%s' (reference '%s') for dependency '%s'", - strings.TrimPrefix(sLocalChartPath, dm.WorkingDir), dpr.Dependency.Repository, dpr.Dependency.Name) + return fmt.Errorf("no chart found at '%s' (reference '%s')", + strings.TrimPrefix(sLocalChartPath, dm.baseDir), dep.Repository) } return err } - ch, err := loader.Load(sLocalChartPath) + constraint, err := semver.NewConstraint(dep.Version) if err != nil { + err = fmt.Errorf("invalid version/constraint format '%s': %w", dep.Version, err) return err } - constraint, err := semver.NewConstraint(dpr.Dependency.Version) + ch, err := loader.Load(sLocalChartPath) if err != nil { - err := fmt.Errorf("dependency '%s' has an invalid version/constraint format: %w", dpr.Dependency.Name, err) - return err + return fmt.Errorf("failed to load chart from '%s' (reference '%s'): %w", + strings.TrimPrefix(sLocalChartPath, dm.baseDir), dep.Repository, err) } - v, err := semver.NewVersion(ch.Metadata.Version) + ver, err := semver.NewVersion(ch.Metadata.Version) if err != nil { return err } - if !constraint.Check(v) { - err = fmt.Errorf("can't get a valid version for dependency '%s'", dpr.Dependency.Name) + if !constraint.Check(ver) { + err = fmt.Errorf("can't get a valid version for constraint '%s'", dep.Version) return err } dm.mu.Lock() - dm.Chart.AddDependency(ch) + dm.chart.AddDependency(ch) dm.mu.Unlock() - return nil } -func (dm *DependencyManager) addRemoteDependency(dpr *DependencyWithRepository) error { - if dpr.Repository == nil { - return fmt.Errorf("no HelmRepository for '%s' dependency", dpr.Dependency.Name) +// addRemoteDependency attempts to resolve and add the given remote chart.Dependency to the chart. +func (dm *DependencyManager) addRemoteDependency(dep *helmchart.Dependency) error { + repo, err := dm.resolveRepository(dep.Repository) + if err != nil { + return err } - if !dpr.Repository.HasIndex() { - if !dpr.Repository.HasCacheFile() { - if _, err := dpr.Repository.CacheIndex(); err != nil { - return err - } - } - if err := dpr.Repository.LoadFromCache(); err != nil { - return err - } + if err = repo.StrategicallyLoadIndex(); err != nil { + return fmt.Errorf("failed to load index for '%s': %w", dep.Name, err) } - chartVer, err := dpr.Repository.Get(dpr.Dependency.Name, dpr.Dependency.Version) + + ver, err := repo.Get(dep.Name, dep.Version) if err != nil { return err } - - res, err := dpr.Repository.DownloadChart(chartVer) + res, err := repo.DownloadChart(ver) if err != nil { - return err + return fmt.Errorf("chart download of version '%s' failed: %w", ver.Version, err) } - ch, err := loader.LoadArchive(res) if err != nil { - return err + return fmt.Errorf("failed to load downloaded archive of version '%s': %w", ver.Version, err) } dm.mu.Lock() - dm.Chart.AddDependency(ch) + dm.chart.AddDependency(ch) dm.mu.Unlock() return nil } -func (dm *DependencyManager) secureLocalChartPath(dep *DependencyWithRepository) (string, error) { - localUrl, err := url.Parse(dep.Dependency.Repository) +// resolveRepository first attempts to resolve the url from the repositories, falling back +// to getChartRepositoryCallback if set. It returns the resolved ChartRepository, or an error. +func (dm *DependencyManager) resolveRepository(url string) (_ *ChartRepository, err error) { + dm.mu.Lock() + defer dm.mu.Unlock() + + nUrl := NormalizeChartRepositoryURL(url) + if _, ok := dm.repositories[nUrl]; !ok { + if dm.getChartRepositoryCallback == nil { + err = fmt.Errorf("no chart repository for URL '%s'", nUrl) + return + } + if dm.repositories == nil { + dm.repositories = map[string]*ChartRepository{} + } + if dm.repositories[nUrl], err = dm.getChartRepositoryCallback(nUrl); err != nil { + err = fmt.Errorf("failed to get chart repository for URL '%s': %w", nUrl, err) + return + } + } + return dm.repositories[nUrl], nil +} + +// secureLocalChartPath returns the secure absolute path of a local dependency. +// It does not allow the dependency's path to be outside the scope of baseDir. +func (dm *DependencyManager) secureLocalChartPath(dep *helmchart.Dependency) (string, error) { + localUrl, err := url.Parse(dep.Repository) if err != nil { return "", fmt.Errorf("failed to parse alleged local chart reference: %w", err) } if localUrl.Scheme != "" && localUrl.Scheme != "file" { - return "", fmt.Errorf("'%s' is not a local chart reference", dep.Dependency.Repository) + return "", fmt.Errorf("'%s' is not a local chart reference", dep.Repository) + } + return securejoin.SecureJoin(dm.baseDir, filepath.Join(dm.path, localUrl.Host, localUrl.Path)) +} + +// collectMissing returns a map with reqs that are missing from current, +// indexed by their alias or name. All dependencies of a chart are present +// if len of returned value == 0. +func collectMissing(current []*helmchart.Chart, reqs []*helmchart.Dependency) map[string]*helmchart.Dependency { + // If the number of dependencies equals the number of requested + // dependencies, there are no missing dependencies + if len(current) == len(reqs) { + return nil + } + + // Build up a map of reqs that are not in current, indexed by their + // alias or name + var missing map[string]*helmchart.Dependency + for _, dep := range reqs { + name := dep.Name + if dep.Alias != "" { + name = dep.Alias + } + // Exclude existing dependencies + found := false + for _, existing := range current { + if existing.Name() == name { + found = true + } + } + if found { + continue + } + if missing == nil { + missing = map[string]*helmchart.Dependency{} + } + missing[name] = dep } - return securejoin.SecureJoin(dm.WorkingDir, filepath.Join(dm.ChartPath, localUrl.Host, localUrl.Path)) + return missing +} + +// isLocalDep returns true if the given chart.Dependency contains a local (file) path reference. +func isLocalDep(dep *helmchart.Dependency) bool { + return dep.Repository == "" || strings.HasPrefix(dep.Repository, "file://") } diff --git a/internal/helm/dependency_manager_test.go b/internal/helm/dependency_manager_test.go index a8e6a0480..e51e6b768 100644 --- a/internal/helm/dependency_manager_test.go +++ b/internal/helm/dependency_manager_test.go @@ -18,12 +18,16 @@ package helm import ( "context" + "errors" "fmt" "os" - "strings" + "path/filepath" + "sync" "testing" + . "github.com/onsi/gomega" helmchart "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/repo" ) @@ -47,177 +51,585 @@ var ( chartVersionV1 = "0.3.0" ) -func TestBuild_WithEmptyDependencies(t *testing.T) { - dm := DependencyManager{ - Dependencies: nil, +func TestDependencyManager_Build(t *testing.T) { + tests := []struct { + name string + baseDir string + path string + repositories map[string]*ChartRepository + getChartRepositoryCallback GetChartRepositoryCallback + want int + wantChartFunc func(g *WithT, c *helmchart.Chart) + wantErr string + }{ + //{ + // // TODO(hidde): add various happy paths + //}, + //{ + // // TODO(hidde): test Chart.lock + //}, + { + name: "build failure returns error", + baseDir: "testdata/charts", + path: "helmchartwithdeps", + wantErr: "failed to add remote dependency 'grafana': no chart repository for URL", + }, + { + name: "no dependencies returns zero", + baseDir: "testdata/charts", + path: "helmchart", + want: 0, + }, } - if err := dm.Build(context.TODO()); err != nil { - t.Errorf("Build() should return nil") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + chart, err := loader.Load(filepath.Join(tt.baseDir, tt.path)) + g.Expect(err).ToNot(HaveOccurred()) + + got, err := NewDependencyManager(chart, tt.baseDir, tt.path). + WithRepositories(tt.repositories). + WithChartRepositoryCallback(tt.getChartRepositoryCallback). + Build(context.TODO()) + + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(got).To(BeZero()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(Equal(tt.want)) + if tt.wantChartFunc != nil { + tt.wantChartFunc(g, chart) + } + }) } } -func TestBuild_WithLocalChart(t *testing.T) { +func TestDependencyManager_build(t *testing.T) { tests := []struct { name string - dep helmchart.Dependency - wantErr bool - errMsg string + deps map[string]*helmchart.Dependency + wantErr string + }{ + { + name: "error remote dependency", + deps: map[string]*helmchart.Dependency{ + "example": {Repository: "https://example.com"}, + }, + wantErr: "failed to add remote dependency", + }, + { + name: "error local dependency", + deps: map[string]*helmchart.Dependency{ + "example": {Repository: "file:///invalid"}, + }, + wantErr: "failed to add remote dependency", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + dm := &DependencyManager{ + baseDir: "testdata/charts", + } + err := dm.build(context.TODO(), tt.deps) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + }) + } +} + +func TestDependencyManager_addLocalDependency(t *testing.T) { + tests := []struct { + name string + dep *helmchart.Dependency + wantErr string + wantFunc func(g *WithT, c *helmchart.Chart) }{ { - name: "valid path", - dep: helmchart.Dependency{ + name: "local dependency", + dep: &helmchart.Dependency{ Name: chartName, Version: chartVersion, - Repository: chartLocalRepository, + Repository: "file://../helmchart", + }, + wantFunc: func(g *WithT, c *helmchart.Chart) { + g.Expect(c.Dependencies()).To(HaveLen(1)) }, }, { - name: "valid path", - dep: helmchart.Dependency{ + name: "version not matching constraint", + dep: &helmchart.Dependency{ Name: chartName, - Alias: "aliased", - Version: chartVersion, - Repository: chartLocalRepository, + Version: "0.2.0", + Repository: "file://../helmchart", }, + wantErr: "can't get a valid version for constraint '0.2.0'", }, { - name: "allowed traversing path", - dep: helmchart.Dependency{ + name: "invalid local reference", + dep: &helmchart.Dependency{ Name: chartName, - Alias: "aliased", Version: chartVersion, - Repository: "file://../../../testdata/charts/helmchartwithdeps/../helmchart", + Repository: "file://../../../absolutely/invalid", }, + wantErr: "no chart found at 'absolutely/invalid'", }, { - name: "invalid path", - dep: helmchart.Dependency{ + name: "invalid chart archive", + dep: &helmchart.Dependency{ Name: chartName, Version: chartVersion, - Repository: "file://../invalid", + Repository: "file://../empty.tgz", }, - wantErr: true, - errMsg: "no chart found at", + wantErr: "failed to load chart from 'empty.tgz'", }, { - name: "illegal traversing path", - dep: helmchart.Dependency{ + name: "invalid constraint", + dep: &helmchart.Dependency{ Name: chartName, - Version: chartVersion, - Repository: "file://../../../../../controllers/testdata/charts/helmchart", + Version: "invalid", + Repository: "file://../helmchart", + }, + wantErr: "invalid version/constraint format 'invalid'", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + dm := &DependencyManager{ + chart: &helmchart.Chart{}, + baseDir: "testdata/charts/", + path: "helmchartwithdeps", + } + + err := dm.addLocalDependency(tt.dep) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + return + } + g.Expect(err).ToNot(HaveOccurred()) + }) + } +} + +func TestDependencyManager_addRemoteDependency(t *testing.T) { + g := NewWithT(t) + + chartB, err := os.ReadFile("testdata/charts/helmchart-0.1.0.tgz") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(chartB).ToNot(BeEmpty()) + + tests := []struct { + name string + repositories map[string]*ChartRepository + dep *helmchart.Dependency + wantFunc func(g *WithT, c *helmchart.Chart) + wantErr string + }{ + { + name: "adds remote dependency", + repositories: map[string]*ChartRepository{ + "https://example.com/": { + Client: &mockGetter{ + response: chartB, + }, + Index: &repo.IndexFile{ + Entries: map[string]repo.ChartVersions{ + chartName: { + &repo.ChartVersion{ + Metadata: &helmchart.Metadata{ + Name: chartName, + Version: chartVersion, + }, + URLs: []string{"https://example.com/foo.tgz"}, + }, + }, + }, + }, + RWMutex: &sync.RWMutex{}, + }, + }, + dep: &helmchart.Dependency{ + Name: chartName, + Repository: "https://example.com", + }, + wantFunc: func(g *WithT, c *helmchart.Chart) { + g.Expect(c.Dependencies()).To(HaveLen(1)) + }, + }, + { + name: "resolve repository error", + repositories: map[string]*ChartRepository{}, + dep: &helmchart.Dependency{ + Repository: "https://example.com", + }, + wantErr: "no chart repository for URL", + }, + { + name: "strategic load error", + repositories: map[string]*ChartRepository{ + "https://example.com/": { + CachePath: "/invalid/cache/path/foo", + RWMutex: &sync.RWMutex{}, + }, + }, + dep: &helmchart.Dependency{ + Repository: "https://example.com", + }, + wantErr: "failed to strategically load index", + }, + { + name: "repository get error", + repositories: map[string]*ChartRepository{ + "https://example.com/": { + Index: &repo.IndexFile{}, + RWMutex: &sync.RWMutex{}, + }, + }, + dep: &helmchart.Dependency{ + Repository: "https://example.com", }, - wantErr: true, - errMsg: "no chart found at", + wantErr: "no chart name found", }, { - name: "invalid version constraint format", - dep: helmchart.Dependency{ + name: "repository version constraint error", + repositories: map[string]*ChartRepository{ + "https://example.com/": { + Index: &repo.IndexFile{ + Entries: map[string]repo.ChartVersions{ + chartName: { + &repo.ChartVersion{ + Metadata: &helmchart.Metadata{ + Name: chartName, + Version: "0.1.0", + }, + }, + }, + }, + }, + RWMutex: &sync.RWMutex{}, + }, + }, + dep: &helmchart.Dependency{ Name: chartName, - Version: "!2.0", - Repository: chartLocalRepository, + Version: "0.2.0", + Repository: "https://example.com", }, - wantErr: true, - errMsg: "has an invalid version/constraint format", + wantErr: fmt.Sprintf("no '%s' chart with version matching '0.2.0' found", chartName), }, { - name: "invalid version", - dep: helmchart.Dependency{ + name: "repository chart download error", + repositories: map[string]*ChartRepository{ + "https://example.com/": { + Index: &repo.IndexFile{ + Entries: map[string]repo.ChartVersions{ + chartName: { + &repo.ChartVersion{ + Metadata: &helmchart.Metadata{ + Name: chartName, + Version: chartVersion, + }, + }, + }, + }, + }, + RWMutex: &sync.RWMutex{}, + }, + }, + dep: &helmchart.Dependency{ Name: chartName, Version: chartVersion, - Repository: chartLocalRepository, + Repository: "https://example.com", }, - wantErr: true, - errMsg: "can't get a valid version for dependency", + wantErr: "chart download of version '0.1.0' failed", + }, + { + name: "chart load error", + repositories: map[string]*ChartRepository{ + "https://example.com/": { + Client: &mockGetter{}, + Index: &repo.IndexFile{ + Entries: map[string]repo.ChartVersions{ + chartName: { + &repo.ChartVersion{ + Metadata: &helmchart.Metadata{ + Name: chartName, + Version: chartVersion, + }, + URLs: []string{"https://example.com/foo.tgz"}, + }, + }, + }, + }, + RWMutex: &sync.RWMutex{}, + }, + }, + dep: &helmchart.Dependency{ + Name: chartName, + Version: chartVersion, + Repository: "https://example.com", + }, + wantErr: "failed to load downloaded archive of version '0.1.0'", }, } - for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := chartFixture - dm := DependencyManager{ - WorkingDir: "./", - ChartPath: "testdata/charts/helmchart", - Chart: &c, - Dependencies: []*DependencyWithRepository{ - { - Dependency: &tt.dep, - Repository: nil, - }, - }, - } + g := NewWithT(t) - err := dm.Build(context.TODO()) - deps := dm.Chart.Dependencies() - - if (err != nil) && tt.wantErr { - if !strings.Contains(err.Error(), tt.errMsg) { - t.Errorf("Build() expected to return error: %s, got: %s", tt.errMsg, err) - } - if len(deps) > 0 { - t.Fatalf("chart expected to have no dependencies registered") - } - return - } else if err != nil { - t.Errorf("Build() not expected to return an error: %s", err) + dm := &DependencyManager{ + chart: &helmchart.Chart{}, + repositories: tt.repositories, + } + err := dm.addRemoteDependency(tt.dep) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) return } + g.Expect(err).ToNot(HaveOccurred()) + if tt.wantFunc != nil { + tt.wantFunc(g, dm.chart) + } + }) + } +} + +func TestDependencyManager_resolveRepository(t *testing.T) { + tests := []struct { + name string + repositories map[string]*ChartRepository + getChartRepositoryCallback GetChartRepositoryCallback + url string + want *ChartRepository + wantRepositories map[string]*ChartRepository + wantErr string + }{ + { + name: "resolves from repositories index", + url: "https://example.com", + repositories: map[string]*ChartRepository{ + "https://example.com/": {URL: "https://example.com"}, + }, + want: &ChartRepository{URL: "https://example.com"}, + }, + { + name: "resolves from callback", + url: "https://example.com", + getChartRepositoryCallback: func(url string) (*ChartRepository, error) { + return &ChartRepository{URL: "https://example.com"}, nil + }, + want: &ChartRepository{URL: "https://example.com"}, + wantRepositories: map[string]*ChartRepository{ + "https://example.com/": {URL: "https://example.com"}, + }, + }, + { + name: "error from callback", + url: "https://example.com", + getChartRepositoryCallback: func(url string) (*ChartRepository, error) { + return nil, errors.New("a very unique error") + }, + wantErr: "a very unique error", + wantRepositories: map[string]*ChartRepository{}, + }, + { + name: "error on not found", + url: "https://example.com", + wantErr: "no chart repository for URL", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) - if len(deps) == 0 { - t.Fatalf("chart expected to have at least one dependency registered") + dm := &DependencyManager{ + repositories: tt.repositories, + getChartRepositoryCallback: tt.getChartRepositoryCallback, } - if deps[0].Metadata.Name != chartName { - t.Errorf("chart dependency has incorrect name, expected: %s, got: %s", chartName, deps[0].Metadata.Name) + + got, err := dm.resolveRepository(tt.url) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(got).To(BeNil()) + return } - if deps[0].Metadata.Version != chartVersion { - t.Errorf("chart dependency has incorrect version, expected: %s, got: %s", chartVersion, deps[0].Metadata.Version) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(Equal(tt.want)) + if tt.wantRepositories != nil { + g.Expect(dm.repositories).To(Equal(tt.wantRepositories)) } }) } } -func TestBuild_WithRemoteChart(t *testing.T) { - chart := chartFixture - b, err := os.ReadFile(helmPackageFile) - if err != nil { - t.Fatal(err) - } - i := repo.NewIndexFile() - i.MustAdd(&helmchart.Metadata{Name: chartName, Version: chartVersion}, fmt.Sprintf("%s-%s.tgz", chartName, chartVersion), "http://example.com/charts", "sha256:1234567890") - mg := mockGetter{response: b} - cr := newChartRepository() - cr.URL = remoteDepFixture.Repository - cr.Index = i - cr.Client = &mg - dm := DependencyManager{ - Chart: &chart, - Dependencies: []*DependencyWithRepository{ - { - Dependency: &remoteDepFixture, - Repository: cr, +func TestDependencyManager_secureLocalChartPath(t *testing.T) { + tests := []struct { + name string + baseDir string + path string + dep *helmchart.Dependency + want string + wantErr string + }{ + { + name: "secure local file path", + baseDir: "/tmp/workdir", + path: "/chart", + dep: &helmchart.Dependency{ + Repository: "../dep", + }, + want: "/tmp/workdir/dep", + }, + { + name: "insecure local file path", + baseDir: "/tmp/workdir", + path: "/", + dep: &helmchart.Dependency{ + Repository: "/../../dep", }, + want: "/tmp/workdir/dep", + }, + { + name: "URL parse error", + dep: &helmchart.Dependency{ + Repository: ": //example.com", + }, + wantErr: "missing protocol scheme", + }, + { + name: "error on URL scheme other than file", + dep: &helmchart.Dependency{ + Repository: "https://example.com", + }, + wantErr: "not a local chart reference", }, } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) - if err := dm.Build(context.TODO()); err != nil { - t.Errorf("Build() expected to not return error: %s", err) + dm := &DependencyManager{ + baseDir: tt.baseDir, + path: tt.path, + } + got, err := dm.secureLocalChartPath(tt.dep) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + return + } + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).ToNot(BeEmpty()) + g.Expect(got).To(Equal(tt.want)) + }) } +} - deps := dm.Chart.Dependencies() - if len(deps) != 1 { - t.Fatalf("chart expected to have one dependency registered") - } - if deps[0].Metadata.Name != chartName { - t.Errorf("chart dependency has incorrect name, expected: %s, got: %s", chartName, deps[0].Metadata.Name) +func Test_collectMissing(t *testing.T) { + tests := []struct { + name string + current []*helmchart.Chart + reqs []*helmchart.Dependency + want map[string]*helmchart.Dependency + }{ + { + name: "one missing", + current: []*helmchart.Chart{}, + reqs: []*helmchart.Dependency{ + {Name: chartName}, + }, + want: map[string]*helmchart.Dependency{ + chartName: {Name: chartName}, + }, + }, + { + name: "alias missing", + current: []*helmchart.Chart{ + { + Metadata: &helmchart.Metadata{ + Name: chartName, + }, + }, + }, + reqs: []*helmchart.Dependency{ + {Name: chartName}, + {Name: chartName, Alias: chartName + "-alias"}, + }, + want: map[string]*helmchart.Dependency{ + chartName + "-alias": {Name: chartName, Alias: chartName + "-alias"}, + }, + }, + { + name: "all current", + current: []*helmchart.Chart{ + { + Metadata: &helmchart.Metadata{ + Name: chartName, + }, + }, + }, + reqs: []*helmchart.Dependency{ + {Name: chartName}, + }, + want: nil, + }, + { + name: "nil", + current: nil, + reqs: nil, + want: nil, + }, } - if deps[0].Metadata.Version != chartVersion { - t.Errorf("chart dependency has incorrect version, expected: %s, got: %s", chartVersion, deps[0].Metadata.Version) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + g.Expect(collectMissing(tt.current, tt.reqs)).To(Equal(tt.want)) + }) + }) } +} - // When repo is not set - dm.Dependencies[0].Repository = nil - if err := dm.Build(context.TODO()); err == nil { - t.Errorf("Build() expected to return error") - } else if !strings.Contains(err.Error(), "is not a local chart reference") { - t.Errorf("Build() expected to return different error, got: %s", err) +func Test_isLocalDep(t *testing.T) { + tests := []struct { + name string + dep *helmchart.Dependency + want bool + }{ + { + name: "file protocol", + dep: &helmchart.Dependency{Repository: "file:///some/path"}, + want: true, + }, + { + name: "empty", + dep: &helmchart.Dependency{Repository: ""}, + want: true, + }, + { + name: "https url", + dep: &helmchart.Dependency{Repository: "https://example.com"}, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + g.Expect(isLocalDep(tt.dep)).To(Equal(tt.want)) + }) } } diff --git a/internal/helm/repository.go b/internal/helm/repository.go index c57df111f..e2446f944 100644 --- a/internal/helm/repository.go +++ b/internal/helm/repository.go @@ -54,6 +54,9 @@ type ChartRepository struct { Options []getter.Option // CachePath is the path of a cached index.yaml for read-only operations. CachePath string + // Cached indicates if the ChartRepository index.yaml has been cached + // to CachePath. + Cached bool // Index contains a loaded chart repository index if not nil. Index *repo.IndexFile // Checksum contains the SHA256 checksum of the loaded chart repository @@ -68,7 +71,6 @@ type ChartRepository struct { // repository URL scheme. It returns an error on URL parsing failures, // or if there is no getter available for the scheme. func NewChartRepository(repositoryURL, cachePath string, providers getter.Providers, opts []getter.Option) (*ChartRepository, error) { - r := newChartRepository() u, err := url.Parse(repositoryURL) if err != nil { return nil, err @@ -77,6 +79,8 @@ func NewChartRepository(repositoryURL, cachePath string, providers getter.Provid if err != nil { return nil, err } + + r := newChartRepository() r.URL = repositoryURL r.CachePath = cachePath r.Client = c @@ -238,7 +242,7 @@ func (r *ChartRepository) LoadFromFile(path string) error { } // CacheIndex attempts to write the index from the remote into a new temporary file -// using DownloadIndex, and sets CachePath. +// using DownloadIndex, and sets CachePath and Cached. // It returns the SHA256 checksum of the downloaded index bytes, or an error. // The caller is expected to handle the garbage collection of CachePath, and to // load the Index separately using LoadFromCache if required. @@ -262,19 +266,40 @@ func (r *ChartRepository) CacheIndex() (string, error) { r.Lock() r.CachePath = f.Name() + r.Cached = true r.Unlock() return hex.EncodeToString(h.Sum(nil)), nil } +// StrategicallyLoadIndex lazy-loads the Index from CachePath using +// LoadFromCache if it does not HasIndex. +// If it not HasCacheFile, a cache attempt is made using CacheIndex +// before continuing to load. +// It returns a boolean indicating if it cached the index before +// loading, or an error. +func (r *ChartRepository) StrategicallyLoadIndex() (err error) { + if r.HasIndex() { + return + } + if !r.HasCacheFile() { + if _, err = r.CacheIndex(); err != nil { + err = fmt.Errorf("failed to strategically load index: %w", err) + return + } + } + if err = r.LoadFromCache(); err != nil { + err = fmt.Errorf("failed to strategically load index: %w", err) + return + } + return +} + // LoadFromCache attempts to load the Index from the configured CachePath. // It returns an error if no CachePath is set, or if the load failed. func (r *ChartRepository) LoadFromCache() error { - r.RLock() if cachePath := r.CachePath; cachePath != "" { - r.RUnlock() return r.LoadFromFile(cachePath) } - r.RUnlock() return fmt.Errorf("no cache path set") } @@ -314,11 +339,34 @@ func (r *ChartRepository) HasCacheFile() bool { return r.CachePath != "" } -// UnloadIndex sets the Index to nil. -func (r *ChartRepository) UnloadIndex() { - if r != nil { - r.Lock() - r.Index = nil - r.Unlock() +// Unload can be used to signal the Go garbage collector the Index can +// be freed from memory if the ChartRepository object is expected to +// continue to exist in the stack for some time. +func (r *ChartRepository) Unload() { + if r == nil { + return } + + r.Lock() + defer r.Unlock() + r.Index = nil +} + +// RemoveCache removes the CachePath if Cached. +func (r *ChartRepository) RemoveCache() error { + if r == nil { + return nil + } + + r.Lock() + defer r.Unlock() + + if r.Cached { + if err := os.Remove(r.CachePath); err != nil && !os.IsNotExist(err) { + return err + } + r.CachePath = "" + r.Cached = false + } + return nil } diff --git a/internal/helm/repository_test.go b/internal/helm/repository_test.go index 95ccc7b80..0d2077dfd 100644 --- a/internal/helm/repository_test.go +++ b/internal/helm/repository_test.go @@ -47,7 +47,8 @@ type mockGetter struct { func (g *mockGetter) Get(url string, _ ...getter.Option) (*bytes.Buffer, error) { g.requestedURL = url - return bytes.NewBuffer(g.response), nil + r := g.response + return bytes.NewBuffer(r), nil } func TestNewChartRepository(t *testing.T) { @@ -402,7 +403,7 @@ func TestChartRepository_CacheIndex(t *testing.T) { g.Expect(sum).To(BeEquivalentTo(expectSum)) } -func TestChartRepository_LoadIndexFromCache(t *testing.T) { +func TestChartRepository_LoadFromCache(t *testing.T) { tests := []struct { name string cachePath string @@ -458,7 +459,7 @@ func TestChartRepository_UnloadIndex(t *testing.T) { r := newChartRepository() g.Expect(r.HasIndex()).To(BeFalse()) r.Index = repo.NewIndexFile() - r.UnloadIndex() + r.Unload() g.Expect(r.Index).To(BeNil()) } diff --git a/internal/helm/testdata/charts/helmchart/values-prod.yaml b/internal/helm/testdata/charts/helmchart/values-prod.yaml new file mode 100644 index 000000000..5ef7832ca --- /dev/null +++ b/internal/helm/testdata/charts/helmchart/values-prod.yaml @@ -0,0 +1 @@ +replicaCount: 2 diff --git a/internal/helm/testdata/charts/helmchartwithdeps/Chart.lock b/internal/helm/testdata/charts/helmchartwithdeps/Chart.lock new file mode 100644 index 000000000..83401ac65 --- /dev/null +++ b/internal/helm/testdata/charts/helmchartwithdeps/Chart.lock @@ -0,0 +1,12 @@ +dependencies: +- name: helmchart + repository: file://../helmchart + version: 0.1.0 +- name: helmchart + repository: file://../helmchart + version: 0.1.0 +- name: grafana + repository: https://grafana.github.io/helm-charts + version: 6.17.4 +digest: sha256:1e41c97e27347f433ff0212bf52c344bc82dd435f70129d15e96cd2c8fcc32bb +generated: "2021-11-02T01:25:59.624290788+01:00" From d23bcbb5db2ae441000b43b4100ef678a6068b9d Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Fri, 5 Nov 2021 13:20:25 +0100 Subject: [PATCH 05/23] controllers: wire ChartRepository in reconciler This wires the `ChartRepository` changes into the reconciler to ensure it works. Signed-off-by: Hidde Beydals --- controllers/helmrepository_controller.go | 57 ++++++++++++++---------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/controllers/helmrepository_controller.go b/controllers/helmrepository_controller.go index b7f8cd516..d7fb57e58 100644 --- a/controllers/helmrepository_controller.go +++ b/controllers/helmrepository_controller.go @@ -17,12 +17,15 @@ limitations under the License. package controllers import ( - "bytes" "context" "fmt" "net/url" "time" + "github.com/fluxcd/pkg/apis/meta" + "github.com/fluxcd/pkg/runtime/events" + "github.com/fluxcd/pkg/runtime/metrics" + "github.com/fluxcd/pkg/runtime/predicates" "github.com/go-logr/logr" "helm.sh/helm/v3/pkg/getter" corev1 "k8s.io/api/core/v1" @@ -37,12 +40,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/yaml" - - "github.com/fluxcd/pkg/apis/meta" - "github.com/fluxcd/pkg/runtime/events" - "github.com/fluxcd/pkg/runtime/metrics" - "github.com/fluxcd/pkg/runtime/predicates" sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" "github.com/fluxcd/source-controller/internal/helm" @@ -198,7 +195,7 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sou clientOpts = append(clientOpts, opts...) } - chartRepo, err := helm.NewChartRepository(repository.Spec.URL, r.Getters, clientOpts) + chartRepo, err := helm.NewChartRepository(repository.Spec.URL, "", r.Getters, clientOpts) if err != nil { switch err.(type) { case *url.Error: @@ -207,22 +204,21 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sou return sourcev1.HelmRepositoryNotReady(repository, sourcev1.IndexationFailedReason, err.Error()), err } } - if err := chartRepo.DownloadIndex(); err != nil { + revision, err := chartRepo.CacheIndex() + if err != nil { err = fmt.Errorf("failed to download repository index: %w", err) return sourcev1.HelmRepositoryNotReady(repository, sourcev1.IndexationFailedReason, err.Error()), err } + defer chartRepo.RemoveCache() - indexBytes, err := yaml.Marshal(&chartRepo.Index) - if err != nil { - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err - } - hash := r.Storage.Checksum(bytes.NewReader(indexBytes)) artifact := r.Storage.NewArtifactFor(repository.Kind, repository.ObjectMeta.GetObjectMeta(), - hash, - fmt.Sprintf("index-%s.yaml", hash)) - // return early on unchanged index - if apimeta.IsStatusConditionTrue(repository.Status.Conditions, meta.ReadyCondition) && repository.GetArtifact().HasRevision(artifact.Revision) { + revision, + fmt.Sprintf("index-%s.yaml", revision)) + + // Return early on unchanged index + if apimeta.IsStatusConditionTrue(repository.Status.Conditions, meta.ReadyCondition) && + repository.GetArtifact().HasRevision(artifact.Revision) { if artifact.URL != repository.GetArtifact().URL { r.Storage.SetArtifactURL(repository.GetArtifact()) repository.Status.URL = r.Storage.SetHostname(repository.Status.URL) @@ -230,14 +226,20 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sou return repository, nil } - // create artifact dir + // Load the cached repository index to ensure it passes validation + if err := chartRepo.LoadFromCache(); err != nil { + return sourcev1.HelmRepositoryNotReady(repository, sourcev1.IndexationFailedReason, err.Error()), err + } + defer chartRepo.Unload() + + // Create artifact dir err = r.Storage.MkdirAll(artifact) if err != nil { err = fmt.Errorf("unable to create repository index directory: %w", err) return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err } - // acquire lock + // Acquire lock unlock, err := r.Storage.Lock(artifact) if err != nil { err = fmt.Errorf("unable to acquire lock: %w", err) @@ -245,13 +247,20 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sou } defer unlock() - // save artifact to storage - if err := r.Storage.AtomicWriteFile(&artifact, bytes.NewReader(indexBytes), 0644); err != nil { - err = fmt.Errorf("unable to write repository index file: %w", err) + // Save artifact to storage + storageTarget := r.Storage.LocalPath(artifact) + if storageTarget == "" { + err := fmt.Errorf("failed to calcalute local storage path to store artifact to") + return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err + } + if err = chartRepo.Index.WriteFile(storageTarget, 0644); err != nil { return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err } + // TODO(hidde): it would be better to make the Storage deal with this + artifact.Checksum = chartRepo.Checksum + artifact.LastUpdateTime = metav1.Now() - // update index symlink + // Update index symlink indexURL, err := r.Storage.Symlink(artifact, "index.yaml") if err != nil { err = fmt.Errorf("storage error: %w", err) From 52459c899da85724ec3b2ef155ce871d0e484756 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Sat, 13 Nov 2021 00:16:59 +0100 Subject: [PATCH 06/23] internal/helm: make ChartBuilder an interface This commit refactors the `ChartBuilder` that used to be a do-it-all struct into an interace with two implementations: - `LocalChartBuilder`: to build charts from a source on the local filesystem, either from a directory or from a packaged chart. - `RemoteChartBuilder`: to build charts from a remote Helm repository index. The new logic within the builders validates the size of the Helm size it works with based on the `Max*Size` global variables in the internal `helm` package, to address the recommendation from the security audit. In addition, changes `ClientOptionsFromSecret` takes now a directory argument which temporary files are placed in, making it easier to perform a garbage collection of the whole directory at the end of a reconcile run. Signed-off-by: Hidde Beydals --- internal/helm/chart.go | 51 +- internal/helm/chart_builder.go | 416 ++++------------ internal/helm/chart_builder_local.go | 190 +++++++ internal/helm/chart_builder_local_test.go | 137 ++++++ internal/helm/chart_builder_remote.go | 199 ++++++++ internal/helm/chart_builder_remote_test.go | 118 +++++ internal/helm/chart_builder_test.go | 543 +-------------------- internal/helm/chart_test.go | 20 +- internal/helm/dependency_manager.go | 175 ++++--- internal/helm/dependency_manager_test.go | 46 +- internal/helm/getter.go | 82 ++-- internal/helm/getter_test.go | 21 +- internal/helm/helm.go | 29 ++ internal/helm/repository.go | 12 +- internal/helm/repository_test.go | 2 +- 15 files changed, 1023 insertions(+), 1018 deletions(-) create mode 100644 internal/helm/chart_builder_local.go create mode 100644 internal/helm/chart_builder_local_test.go create mode 100644 internal/helm/chart_builder_remote.go create mode 100644 internal/helm/chart_builder_remote_test.go create mode 100644 internal/helm/helm.go diff --git a/internal/helm/chart.go b/internal/helm/chart.go index dcc868c1d..4f89cab61 100644 --- a/internal/helm/chart.go +++ b/internal/helm/chart.go @@ -19,6 +19,7 @@ package helm import ( "archive/tar" "bufio" + "bytes" "compress/gzip" "errors" "fmt" @@ -35,30 +36,35 @@ import ( ) // OverwriteChartDefaultValues overwrites the chart default values file with the given data. -func OverwriteChartDefaultValues(chart *helmchart.Chart, data []byte) (bool, error) { - // Read override values file data - values, err := chartutil.ReadValues(data) - if err != nil { - return false, fmt.Errorf("failed to parse provided override values file data") +func OverwriteChartDefaultValues(chart *helmchart.Chart, vals chartutil.Values) (bool, error) { + if vals == nil { + return false, nil + } + + var bVals bytes.Buffer + if len(vals) > 0 { + if err := vals.Encode(&bVals); err != nil { + return false, err + } } // Replace current values file in Raw field for _, f := range chart.Raw { if f.Name == chartutil.ValuesfileName { // Do nothing if contents are equal - if reflect.DeepEqual(f.Data, data) { + if reflect.DeepEqual(f.Data, bVals.Bytes()) { return false, nil } // Replace in Files field for _, f := range chart.Files { if f.Name == chartutil.ValuesfileName { - f.Data = data + f.Data = bVals.Bytes() } } - f.Data = data - chart.Values = values + f.Data = bVals.Bytes() + chart.Values = vals.AsMap() return true, nil } } @@ -100,7 +106,21 @@ func LoadChartMetadataFromDir(dir string) (*helmchart.Metadata, error) { m.APIVersion = helmchart.APIVersionV1 } - b, err = os.ReadFile(filepath.Join(dir, "requirements.yaml")) + fp := filepath.Join(dir, "requirements.yaml") + stat, err := os.Stat(fp) + if (err != nil && !errors.Is(err, os.ErrNotExist)) || stat != nil { + if err != nil { + return nil, err + } + if stat.IsDir() { + return nil, fmt.Errorf("'%s' is a directory", stat.Name()) + } + if stat.Size() > MaxChartFileSize { + return nil, fmt.Errorf("size of '%s' exceeds '%d' limit", stat.Name(), MaxChartFileSize) + } + } + + b, err = os.ReadFile(fp) if err != nil && !errors.Is(err, os.ErrNotExist) { return nil, err } @@ -115,6 +135,17 @@ func LoadChartMetadataFromDir(dir string) (*helmchart.Metadata, error) { // LoadChartMetadataFromArchive loads the chart.Metadata from the "Chart.yaml" file in the archive at the given path. // It takes "requirements.yaml" files into account, and is therefore compatible with the chart.APIVersionV1 format. func LoadChartMetadataFromArchive(archive string) (*helmchart.Metadata, error) { + stat, err := os.Stat(archive) + if err != nil || stat.IsDir() { + if err == nil { + err = fmt.Errorf("'%s' is a directory", stat.Name()) + } + return nil, err + } + if stat.Size() > MaxChartSize { + return nil, fmt.Errorf("size of chart '%s' exceeds '%d' limit", stat.Name(), MaxChartSize) + } + f, err := os.Open(archive) if err != nil { return nil, err diff --git a/internal/helm/chart_builder.go b/internal/helm/chart_builder.go index 7b90cba81..4177983c6 100644 --- a/internal/helm/chart_builder.go +++ b/internal/helm/chart_builder.go @@ -22,338 +22,145 @@ import ( "os" "path/filepath" "strings" - "sync" - securejoin "github.com/cyphar/filepath-securejoin" "github.com/fluxcd/source-controller/internal/fs" helmchart "helm.sh/helm/v3/pkg/chart" - "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/chartutil" - "sigs.k8s.io/yaml" - - "github.com/fluxcd/pkg/runtime/transform" ) -// ChartBuilder aims to efficiently build a Helm chart from a directory or packaged chart. -// It avoids or delays loading the chart into memory in full, working with chart.Metadata -// as much as it can, and returns early (by copying over the already packaged source chart) -// if no modifications were made during the build process. -type ChartBuilder struct { - // baseDir is the chroot for the chart builder when path isDir. - // It must be (a higher) relative to path. File references (during e.g. - // value file merge operations) are not allowed to traverse out of it. - baseDir string - - // path is the file or directory path to a chart source. - path string - - // chart holds a (partly) loaded chart.Chart, it contains at least the - // chart.Metadata, which may expand to the full chart.Chart if required - // for Build operations. - chart *helmchart.Chart - - // valueFiles holds a list of path references of valueFiles that should be - // merged and packaged as a single "values.yaml" during Build. - valueFiles []string - - // repositories holds an index of repository URLs and their ChartRepository. - // They are used to configure a DependencyManager for missing chart dependencies - // if isDir is true. - repositories map[string]*ChartRepository - - // getChartRepositoryCallback is used to configure a DependencyManager for - // missing chart dependencies if isDir is true. - getChartRepositoryCallback GetChartRepositoryCallback - - mu sync.Mutex -} - -// NewChartBuilder constructs a new ChartBuilder for the given chart path. -// It returns an error if no chart.Metadata can be loaded from the path. -func NewChartBuilder(path string) (*ChartBuilder, error) { - metadata, err := LoadChartMetadata(path) - if err != nil { - return nil, fmt.Errorf("could not create new chart builder: %w", err) - } - return &ChartBuilder{ - path: path, - chart: &helmchart.Chart{ - Metadata: metadata, - }, - }, nil +// ChartReference holds information to locate a chart. +type ChartReference interface { + // Validate returns an error if the ChartReference is not valid according + // to the spec of the interface implementation. + Validate() error } -// WithBaseDir configures the base dir on the ChartBuilder. -func (b *ChartBuilder) WithBaseDir(p string) *ChartBuilder { - b.mu.Lock() - b.baseDir = p - b.mu.Unlock() - return b -} - -// WithValueFiles appends the given paths to the ChartBuilder's valueFiles. -func (b *ChartBuilder) WithValueFiles(path ...string) *ChartBuilder { - b.mu.Lock() - b.valueFiles = append(b.valueFiles, path...) - b.mu.Unlock() - return b -} - -// WithChartRepository indexes the given ChartRepository by the NormalizeChartRepositoryURL, -// used to configure the DependencyManager if the chart is not packaged. -func (b *ChartBuilder) WithChartRepository(url string, index *ChartRepository) *ChartBuilder { - b.mu.Lock() - b.repositories[NormalizeChartRepositoryURL(url)] = index - b.mu.Unlock() - return b -} - -// WithChartRepositoryCallback configures the GetChartRepositoryCallback used by the -// DependencyManager if the chart is not packaged. -func (b *ChartBuilder) WithChartRepositoryCallback(c GetChartRepositoryCallback) *ChartBuilder { - b.mu.Lock() - b.getChartRepositoryCallback = c - b.mu.Unlock() - return b -} - -// ChartBuildResult contains the ChartBuilder result, including build specific -// information about the chart. -type ChartBuildResult struct { - // SourceIsDir indicates if the chart was build from a directory. - SourceIsDir bool - // Path contains the absolute path to the packaged chart. +// LocalChartReference contains sufficient information to locate a chart on the +// local filesystem. +type LocalChartReference struct { + // BaseDir used as chroot during build operations. + // File references are not allowed to traverse outside it. + BaseDir string + // Path of the chart on the local filesystem. Path string - // ValuesOverwrite holds a structured map with the merged values used - // to overwrite chart default "values.yaml". - ValuesOverwrite map[string]interface{} - // CollectedDependencies contains the number of missing local and remote - // dependencies that were collected by the DependencyManager before building - // the chart. - CollectedDependencies int - // Packaged indicates if the ChartBuilder has packaged the chart. - // This can for example be false if SourceIsDir is false and ValuesOverwrite - // is nil, which makes the ChartBuilder copy the chart source to Path without - // making any modifications. - Packaged bool } -// String returns the Path of the ChartBuildResult. -func (b *ChartBuildResult) String() string { - if b != nil { - return b.Path +// Validate returns an error if the LocalChartReference does not have +// a Path set. +func (r LocalChartReference) Validate() error { + if r.Path == "" { + return fmt.Errorf("no path set for local chart reference") } - return "" + return nil } -// Build attempts to build a new chart using ChartBuilder configuration, -// writing it to the provided path. -// It returns a ChartBuildResult containing all information about the resulting chart, -// or an error. -func (b *ChartBuilder) Build(ctx context.Context, p string) (_ *ChartBuildResult, err error) { - b.mu.Lock() - defer b.mu.Unlock() - - if b.chart == nil { - err = fmt.Errorf("chart build failed: no initial chart (metadata) loaded") - return - } - if b.path == "" { - err = fmt.Errorf("chart build failed: no path set") - return - } - - result := &ChartBuildResult{} - result.SourceIsDir = pathIsDir(b.path) - result.Path = p - - // Merge chart values - if err = b.mergeValues(result); err != nil { - err = fmt.Errorf("chart build failed: %w", err) - return - } - - // Ensure chart has all dependencies - if err = b.buildDependencies(ctx, result); err != nil { - err = fmt.Errorf("chart build failed: %w", err) - return - } - - // Package (or copy) chart - if err = b.packageChart(result); err != nil { - err = fmt.Errorf("chart package failed: %w", err) - return - } - return result, nil +// RemoteChartReference contains sufficient information to look up a chart in +// a ChartRepository. +type RemoteChartReference struct { + // Name of the chart. + Name string + // Version of the chart. + // Can be a Semver range, or empty for latest. + Version string } -// load lazy-loads chart.Chart into chart from the set path, it replaces any previously set -// chart.Metadata shim. -func (b *ChartBuilder) load() (err error) { - if b.chart == nil || len(b.chart.Files) <= 0 { - if b.path == "" { - return fmt.Errorf("failed to load chart: path not set") - } - chart, err := loader.Load(b.path) - if err != nil { - return fmt.Errorf("failed to load chart: %w", err) - } - b.chart = chart +// Validate returns an error if the RemoteChartReference does not have +// a Name set. +func (r RemoteChartReference) Validate() error { + if r.Name == "" { + return fmt.Errorf("no name set for remote chart reference") } - return + return nil } -// buildDependencies builds the missing dependencies for a chart from a directory. -// Using the chart using a NewDependencyManager and the configured repositories -// and getChartRepositoryCallback -// It returns the number of dependencies it collected, or an error. -func (b *ChartBuilder) buildDependencies(ctx context.Context, result *ChartBuildResult) (err error) { - if !result.SourceIsDir { - return - } - - if err = b.load(); err != nil { - err = fmt.Errorf("failed to ensure chart has no missing dependencies: %w", err) - return +// ChartBuilder is capable of building a (specific) ChartReference. +type ChartBuilder interface { + // Build builds and packages a Helm chart with the given ChartReference + // and BuildOptions and writes it to p. It returns the ChartBuild result, + // or an error. It may return an error for unsupported ChartReference + // implementations. + Build(ctx context.Context, ref ChartReference, p string, opts BuildOptions) (*ChartBuild, error) +} + +// BuildOptions provides a list of options for ChartBuilder.Build. +type BuildOptions struct { + // VersionMetadata can be set to SemVer build metadata as defined in + // the spec, and is included during packaging. + // Ref: https://semver.org/#spec-item-10 + VersionMetadata string + // ValueFiles can be set to a list of relative paths, used to compose + // and overwrite an alternative default "values.yaml" for the chart. + ValueFiles []string + // CachedChart can be set to the absolute path of a chart stored on + // the local filesystem, and is used for simple validation by metadata + // comparisons. + CachedChart string + // Force can be set to force the build of the chart, for example + // because the list of ValueFiles has changed. + Force bool +} + +// GetValueFiles returns BuildOptions.ValueFiles, except if it equals +// "values.yaml", which returns nil. +func (o BuildOptions) GetValueFiles() []string { + if len(o.ValueFiles) == 1 && filepath.Clean(o.ValueFiles[0]) == filepath.Clean(chartutil.ValuesfileName) { + return nil } - - dm := NewDependencyManager(b.chart, b.baseDir, strings.TrimLeft(b.path, b.baseDir)). - WithRepositories(b.repositories). - WithChartRepositoryCallback(b.getChartRepositoryCallback) - - result.CollectedDependencies, err = dm.Build(ctx) - return + return o.ValueFiles } -// mergeValues strategically merges the valueFiles, it merges using mergeFileValues -// or mergeChartValues depending on if the chart is sourced from a package or directory. -// Ir only calls load to propagate the chart if required by the strategy. -// It returns the merged values, or an error. -func (b *ChartBuilder) mergeValues(result *ChartBuildResult) (err error) { - if len(b.valueFiles) == 0 { - return - } +// ChartBuild contains the ChartBuilder.Build result, including specific +// information about the built chart like ResolvedDependencies. +type ChartBuild struct { + // Path is the absolute path to the packaged chart. + Path string + // Name of the packaged chart. + Name string + // Version of the packaged chart. + Version string + // ValueFiles is the list of files used to compose the chart's + // default "values.yaml". + ValueFiles []string + // ResolvedDependencies is the number of local and remote dependencies + // collected by the DependencyManager before building the chart. + ResolvedDependencies int + // Packaged indicates if the ChartBuilder has packaged the chart. + // This can for example be false if ValueFiles is empty and the chart + // source was already packaged. + Packaged bool +} - if result.SourceIsDir { - result.ValuesOverwrite, err = mergeFileValues(b.baseDir, b.valueFiles) - if err != nil { - err = fmt.Errorf("failed to merge value files: %w", err) - } - return +// Summary returns a human-readable summary of the ChartBuild. +func (b *ChartBuild) Summary() string { + if b == nil { + return "no chart build" } - // Values equal to default - if len(b.valueFiles) == 1 && b.valueFiles[0] == chartutil.ValuesfileName { - return - } + var s strings.Builder - if err = b.load(); err != nil { - err = fmt.Errorf("failed to merge chart values: %w", err) - return + action := "Fetched" + if b.Packaged { + action = "Packaged" } + s.WriteString(fmt.Sprintf("%s '%s' chart with version '%s'.", action, b.Name, b.Version)) - if result.ValuesOverwrite, err = mergeChartValues(b.chart, b.valueFiles); err != nil { - err = fmt.Errorf("failed to merge chart values: %w", err) - return + if b.Packaged && b.ResolvedDependencies > 0 { + s.WriteString(fmt.Sprintf(" Resolved %d dependencies before packaging.", b.ResolvedDependencies)) } - return nil -} -// packageChart determines if it should copyFileToPath or packageToPath -// based on the provided result. It sets Packaged on ChartBuildResult to -// true if packageToPath is successful. -func (b *ChartBuilder) packageChart(result *ChartBuildResult) error { - // If we are not building from a directory, and we do not have any - // replacement values, we can copy over the already packaged source - // chart without making any modifications - if !result.SourceIsDir && len(result.ValuesOverwrite) == 0 { - if err := copyFileToPath(b.path, result.Path); err != nil { - return fmt.Errorf("chart build failed: %w", err) - } - return nil + if len(b.ValueFiles) > 0 { + s.WriteString(fmt.Sprintf(" Merged %v value files into default chart values.", b.ValueFiles)) } - // Package chart to a new temporary directory - if err := packageToPath(b.chart, result.Path); err != nil { - return fmt.Errorf("chart build failed: %w", err) - } - result.Packaged = true - return nil + return s.String() } -// mergeChartValues merges the given chart.Chart Files paths into a single "values.yaml" map. -// It returns the merge result, or an error. -func mergeChartValues(chart *helmchart.Chart, paths []string) (map[string]interface{}, error) { - mergedValues := make(map[string]interface{}) - for _, p := range paths { - cfn := filepath.Clean(p) - if cfn == chartutil.ValuesfileName { - mergedValues = transform.MergeMaps(mergedValues, chart.Values) - continue - } - var b []byte - for _, f := range chart.Files { - if f.Name == cfn { - b = f.Data - break - } - } - if b == nil { - return nil, fmt.Errorf("no values file found at path '%s'", p) - } - values := make(map[string]interface{}) - if err := yaml.Unmarshal(b, &values); err != nil { - return nil, fmt.Errorf("unmarshaling values from '%s' failed: %w", p, err) - } - mergedValues = transform.MergeMaps(mergedValues, values) - } - return mergedValues, nil -} - -// mergeFileValues merges the given value file paths into a single "values.yaml" map. -// The provided (relative) paths may not traverse outside baseDir. It returns the merge -// result, or an error. -func mergeFileValues(baseDir string, paths []string) (map[string]interface{}, error) { - mergedValues := make(map[string]interface{}) - for _, p := range paths { - secureP, err := securejoin.SecureJoin(baseDir, p) - if err != nil { - return nil, err - } - if f, err := os.Stat(secureP); os.IsNotExist(err) || !f.Mode().IsRegular() { - return nil, fmt.Errorf("no values file found at path '%s' (reference '%s')", - strings.TrimPrefix(secureP, baseDir), p) - } - b, err := os.ReadFile(secureP) - if err != nil { - return nil, fmt.Errorf("could not read values from file '%s': %w", p, err) - } - values := make(map[string]interface{}) - err = yaml.Unmarshal(b, &values) - if err != nil { - return nil, fmt.Errorf("unmarshaling values from '%s' failed: %w", p, err) - } - mergedValues = transform.MergeMaps(mergedValues, values) - } - return mergedValues, nil -} - -// copyFileToPath attempts to copy in to out. It returns an error if out already exists. -func copyFileToPath(in, out string) error { - o, err := os.Create(out) - if err != nil { - return fmt.Errorf("failed to create copy target: %w", err) - } - defer o.Close() - i, err := os.Open(in) - if err != nil { - return fmt.Errorf("failed to open file to copy from: %w", err) - } - defer i.Close() - if _, err := o.ReadFrom(i); err != nil { - return fmt.Errorf("failed to read from source during copy: %w", err) +// String returns the Path of the ChartBuild. +func (b *ChartBuild) String() string { + if b != nil { + return b.Path } - return nil + return "" } // packageToPath attempts to package the given chart.Chart to the out filepath. @@ -368,17 +175,8 @@ func packageToPath(chart *helmchart.Chart, out string) error { if err != nil { return fmt.Errorf("failed to package chart: %w", err) } - return fs.RenameWithFallback(p, out) -} - -// pathIsDir returns a boolean indicating if the given path points to a directory. -// In case os.Stat on the given path returns an error it returns false as well. -func pathIsDir(p string) bool { - if p == "" { - return false + if err = fs.RenameWithFallback(p, out); err != nil { + return fmt.Errorf("failed to write chart to file: %w", err) } - if i, err := os.Stat(p); err != nil || !i.IsDir() { - return false - } - return true + return nil } diff --git a/internal/helm/chart_builder_local.go b/internal/helm/chart_builder_local.go new file mode 100644 index 000000000..13e5dbe9c --- /dev/null +++ b/internal/helm/chart_builder_local.go @@ -0,0 +1,190 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helm + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/Masterminds/semver/v3" + securejoin "github.com/cyphar/filepath-securejoin" + "github.com/fluxcd/pkg/runtime/transform" + "helm.sh/helm/v3/pkg/chart/loader" + "sigs.k8s.io/yaml" +) + +type localChartBuilder struct { + dm *DependencyManager +} + +// NewLocalChartBuilder returns a ChartBuilder capable of building a Helm +// chart with a LocalChartReference. For chart references pointing to a +// directory, the DependencyManager is used to resolve missing local and +// remote dependencies. +func NewLocalChartBuilder(dm *DependencyManager) ChartBuilder { + return &localChartBuilder{ + dm: dm, + } +} + +func (b *localChartBuilder) Build(ctx context.Context, ref ChartReference, p string, opts BuildOptions) (*ChartBuild, error) { + localRef, ok := ref.(LocalChartReference) + if !ok { + return nil, fmt.Errorf("expected local chart reference") + } + + if err := ref.Validate(); err != nil { + return nil, err + } + + // Load the chart metadata from the LocalChartReference to ensure it points + // to a chart + curMeta, err := LoadChartMetadata(localRef.Path) + if err != nil { + return nil, err + } + + result := &ChartBuild{} + result.Name = curMeta.Name + + // Set build specific metadata if instructed + result.Version = curMeta.Version + if opts.VersionMetadata != "" { + ver, err := semver.NewVersion(curMeta.Version) + if err != nil { + return nil, fmt.Errorf("failed to parse chart version from metadata as SemVer: %w", err) + } + if *ver, err = ver.SetMetadata(opts.VersionMetadata); err != nil { + return nil, fmt.Errorf("failed to set metadata on chart version: %w", err) + } + result.Version = ver.String() + } + + // If all the following is true, we do not need to package the chart: + // Chart version from metadata matches chart version for ref + // BuildOptions.Force is False + if opts.CachedChart != "" && !opts.Force { + if curMeta, err = LoadChartMetadataFromArchive(opts.CachedChart); err == nil && result.Version == curMeta.Version { + result.Path = opts.CachedChart + result.ValueFiles = opts.ValueFiles + return result, nil + } + } + + // If the chart at the path is already packaged and no custom value files + // options are set, we can copy the chart without making modifications + isChartDir := pathIsDir(localRef.Path) + if !isChartDir && len(opts.GetValueFiles()) == 0 { + if err := copyFileToPath(localRef.Path, p); err != nil { + return nil, err + } + result.Path = p + return result, nil + } + + // Merge chart values, if instructed + var mergedValues map[string]interface{} + if len(opts.GetValueFiles()) > 0 { + if mergedValues, err = mergeFileValues(localRef.BaseDir, opts.ValueFiles); err != nil { + return nil, fmt.Errorf("failed to merge value files: %w", err) + } + } + + // At this point we are certain we need to load the chart; + // either to package it because it originates from a directory, + // or because we have merged values and need to repackage + chart, err := loader.Load(localRef.Path) + if err != nil { + return nil, err + } + // Set earlier resolved version (with metadata) + chart.Metadata.Version = result.Version + + // Overwrite default values with merged values, if any + if ok, err = OverwriteChartDefaultValues(chart, mergedValues); ok || err != nil { + if err != nil { + return nil, err + } + result.ValueFiles = opts.GetValueFiles() + } + + // Ensure dependencies are fetched if building from a directory + if isChartDir { + if b.dm == nil { + return nil, fmt.Errorf("local chart builder requires dependency manager for unpackaged charts") + } + if result.ResolvedDependencies, err = b.dm.Build(ctx, ref, chart); err != nil { + return nil, err + } + } + + // Package the chart + if err = packageToPath(chart, p); err != nil { + return nil, err + } + result.Path = p + result.Packaged = true + return result, nil +} + +// mergeFileValues merges the given value file paths into a single "values.yaml" map. +// The provided (relative) paths may not traverse outside baseDir. It returns the merge +// result, or an error. +func mergeFileValues(baseDir string, paths []string) (map[string]interface{}, error) { + mergedValues := make(map[string]interface{}) + for _, p := range paths { + secureP, err := securejoin.SecureJoin(baseDir, p) + if err != nil { + return nil, err + } + if f, err := os.Stat(secureP); os.IsNotExist(err) || !f.Mode().IsRegular() { + return nil, fmt.Errorf("no values file found at path '%s' (reference '%s')", + strings.TrimPrefix(secureP, baseDir), p) + } + b, err := os.ReadFile(secureP) + if err != nil { + return nil, fmt.Errorf("could not read values from file '%s': %w", p, err) + } + values := make(map[string]interface{}) + err = yaml.Unmarshal(b, &values) + if err != nil { + return nil, fmt.Errorf("unmarshaling values from '%s' failed: %w", p, err) + } + mergedValues = transform.MergeMaps(mergedValues, values) + } + return mergedValues, nil +} + +// copyFileToPath attempts to copy in to out. It returns an error if out already exists. +func copyFileToPath(in, out string) error { + o, err := os.Create(out) + if err != nil { + return fmt.Errorf("failed to create copy target: %w", err) + } + defer o.Close() + i, err := os.Open(in) + if err != nil { + return fmt.Errorf("failed to open file to copy from: %w", err) + } + defer i.Close() + if _, err := o.ReadFrom(i); err != nil { + return fmt.Errorf("failed to read from source during copy: %w", err) + } + return nil +} diff --git a/internal/helm/chart_builder_local_test.go b/internal/helm/chart_builder_local_test.go new file mode 100644 index 000000000..c2f16d694 --- /dev/null +++ b/internal/helm/chart_builder_local_test.go @@ -0,0 +1,137 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helm + +import ( + "os" + "path/filepath" + "testing" + + . "github.com/onsi/gomega" + helmchart "helm.sh/helm/v3/pkg/chart" +) + +func Test_mergeFileValues(t *testing.T) { + tests := []struct { + name string + files []*helmchart.File + paths []string + want map[string]interface{} + wantErr string + }{ + { + name: "merges values from files", + files: []*helmchart.File{ + {Name: "a.yaml", Data: []byte("a: b")}, + {Name: "b.yaml", Data: []byte("b: c")}, + {Name: "c.yaml", Data: []byte("b: d")}, + }, + paths: []string{"a.yaml", "b.yaml", "c.yaml"}, + want: map[string]interface{}{ + "a": "b", + "b": "d", + }, + }, + { + name: "illegal traverse", + paths: []string{"../../../traversing/illegally/a/p/a/b"}, + wantErr: "no values file found at path '/traversing/illegally/a/p/a/b'", + }, + { + name: "unmarshal error", + files: []*helmchart.File{ + {Name: "invalid", Data: []byte("abcd")}, + }, + paths: []string{"invalid"}, + wantErr: "unmarshaling values from 'invalid' failed", + }, + { + name: "error on invalid path", + paths: []string{"a.yaml"}, + wantErr: "no values file found at path '/a.yaml'", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + baseDir, err := os.MkdirTemp("", "merge-file-values-*") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(baseDir) + + for _, f := range tt.files { + g.Expect(os.WriteFile(filepath.Join(baseDir, f.Name), f.Data, 0644)).To(Succeed()) + } + + got, err := mergeFileValues(baseDir, tt.paths) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(got).To(BeNil()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(Equal(tt.want)) + }) + } +} + +func Test_copyFileToPath(t *testing.T) { + tests := []struct { + name string + in string + wantErr string + }{ + { + name: "copies input file", + in: "testdata/local-index.yaml", + }, + { + name: "invalid input file", + in: "testdata/invalid.tgz", + wantErr: "failed to open file to copy from", + }, + { + name: "invalid input directory", + in: "testdata/charts", + wantErr: "failed to read from source during copy", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + out := tmpFile("copy-0.1.0", ".tgz") + defer os.RemoveAll(out) + err := copyFileToPath(tt.in, out) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(out).To(BeARegularFile()) + f1, err := os.ReadFile(tt.in) + g.Expect(err).ToNot(HaveOccurred()) + f2, err := os.ReadFile(out) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(f2).To(Equal(f1)) + }) + } +} diff --git a/internal/helm/chart_builder_remote.go b/internal/helm/chart_builder_remote.go new file mode 100644 index 000000000..18ff317d8 --- /dev/null +++ b/internal/helm/chart_builder_remote.go @@ -0,0 +1,199 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helm + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/Masterminds/semver/v3" + "github.com/fluxcd/pkg/runtime/transform" + "github.com/fluxcd/source-controller/internal/fs" + helmchart "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/chartutil" + "sigs.k8s.io/yaml" +) + +type remoteChartBuilder struct { + remote *ChartRepository +} + +// NewRemoteChartBuilder returns a ChartBuilder capable of building a Helm +// chart with a RemoteChartReference from the given ChartRepository. +func NewRemoteChartBuilder(repository *ChartRepository) ChartBuilder { + return &remoteChartBuilder{ + remote: repository, + } +} + +func (b *remoteChartBuilder) Build(_ context.Context, ref ChartReference, p string, opts BuildOptions) (*ChartBuild, error) { + remoteRef, ok := ref.(RemoteChartReference) + if !ok { + return nil, fmt.Errorf("expected remote chart reference") + } + + if err := ref.Validate(); err != nil { + return nil, err + } + + if err := b.remote.LoadFromCache(); err != nil { + return nil, fmt.Errorf("could not load repository index for remote chart reference: %w", err) + } + defer b.remote.Unload() + + // Get the current version for the RemoteChartReference + cv, err := b.remote.Get(remoteRef.Name, remoteRef.Version) + if err != nil { + return nil, fmt.Errorf("failed to get chart version for remote reference: %w", err) + } + + result := &ChartBuild{} + result.Name = cv.Name + result.Version = cv.Version + // Set build specific metadata if instructed + if opts.VersionMetadata != "" { + ver, err := semver.NewVersion(result.Version) + if err != nil { + return nil, err + } + if *ver, err = ver.SetMetadata(opts.VersionMetadata); err != nil { + return nil, err + } + result.Version = ver.String() + } + + // If all the following is true, we do not need to download and/or build the chart: + // Chart version from metadata matches chart version for ref + // BuildOptions.Force is False + if opts.CachedChart != "" && !opts.Force { + if curMeta, err := LoadChartMetadataFromArchive(opts.CachedChart); err == nil && result.Version == curMeta.Version { + result.Path = opts.CachedChart + result.ValueFiles = opts.GetValueFiles() + return result, nil + } + } + + // Download the package for the resolved version + res, err := b.remote.DownloadChart(cv) + if err != nil { + return nil, fmt.Errorf("failed to download chart for remote reference: %w", err) + } + + // Use literal chart copy from remote if no custom value files options are set + if len(opts.GetValueFiles()) == 0 { + if err = validatePackageAndWriteToPath(res, p); err != nil { + return nil, err + } + result.Path = p + return result, nil + } + + // Load the chart and merge chart values + var chart *helmchart.Chart + if chart, err = loader.LoadArchive(res); err != nil { + return nil, fmt.Errorf("failed to load downloaded chart: %w", err) + } + + mergedValues, err := mergeChartValues(chart, opts.ValueFiles) + if err != nil { + return nil, fmt.Errorf("failed to merge chart values: %w", err) + } + // Overwrite default values with merged values, if any + if ok, err = OverwriteChartDefaultValues(chart, mergedValues); ok || err != nil { + if err != nil { + return nil, err + } + result.ValueFiles = opts.GetValueFiles() + } + + // Package the chart with the custom values + if err = packageToPath(chart, p); err != nil { + return nil, err + } + result.Path = p + result.Packaged = true + return result, nil +} + +// mergeChartValues merges the given chart.Chart Files paths into a single "values.yaml" map. +// It returns the merge result, or an error. +func mergeChartValues(chart *helmchart.Chart, paths []string) (map[string]interface{}, error) { + mergedValues := make(map[string]interface{}) + for _, p := range paths { + cfn := filepath.Clean(p) + if cfn == chartutil.ValuesfileName { + mergedValues = transform.MergeMaps(mergedValues, chart.Values) + continue + } + var b []byte + for _, f := range chart.Files { + if f.Name == cfn { + b = f.Data + break + } + } + if b == nil { + return nil, fmt.Errorf("no values file found at path '%s'", p) + } + values := make(map[string]interface{}) + if err := yaml.Unmarshal(b, &values); err != nil { + return nil, fmt.Errorf("unmarshaling values from '%s' failed: %w", p, err) + } + mergedValues = transform.MergeMaps(mergedValues, values) + } + return mergedValues, nil +} + +// validatePackageAndWriteToPath atomically writes the packaged chart from reader +// to out while validating it by loading the chart metadata from the archive. +func validatePackageAndWriteToPath(reader io.Reader, out string) error { + tmpFile, err := os.CreateTemp("", filepath.Base(out)) + if err != nil { + return fmt.Errorf("failed to create temporary file for chart: %w", err) + } + defer os.Remove(tmpFile.Name()) + if _, err = tmpFile.ReadFrom(reader); err != nil { + _ = tmpFile.Close() + return fmt.Errorf("failed to write chart to file: %w", err) + } + if err = tmpFile.Close(); err != nil { + return err + } + if _, err = LoadChartMetadataFromArchive(tmpFile.Name()); err != nil { + return fmt.Errorf("failed to load chart metadata from written chart: %w", err) + } + if err = fs.RenameWithFallback(tmpFile.Name(), out); err != nil { + return fmt.Errorf("failed to write chart to file: %w", err) + } + return nil +} + +// pathIsDir returns a boolean indicating if the given path points to a directory. +// In case os.Stat on the given path returns an error it returns false as well. +func pathIsDir(p string) bool { + if p == "" { + return false + } + if i, err := os.Stat(p); err != nil || !i.IsDir() { + return false + } + return true +} diff --git a/internal/helm/chart_builder_remote_test.go b/internal/helm/chart_builder_remote_test.go new file mode 100644 index 000000000..260bcbce1 --- /dev/null +++ b/internal/helm/chart_builder_remote_test.go @@ -0,0 +1,118 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helm + +import ( + "testing" + + . "github.com/onsi/gomega" + helmchart "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chartutil" +) + +func Test_mergeChartValues(t *testing.T) { + tests := []struct { + name string + chart *helmchart.Chart + paths []string + want map[string]interface{} + wantErr string + }{ + { + name: "merges values", + chart: &helmchart.Chart{ + Files: []*helmchart.File{ + {Name: "a.yaml", Data: []byte("a: b")}, + {Name: "b.yaml", Data: []byte("b: c")}, + {Name: "c.yaml", Data: []byte("b: d")}, + }, + }, + paths: []string{"a.yaml", "b.yaml", "c.yaml"}, + want: map[string]interface{}{ + "a": "b", + "b": "d", + }, + }, + { + name: "uses chart values", + chart: &helmchart.Chart{ + Files: []*helmchart.File{ + {Name: "c.yaml", Data: []byte("b: d")}, + }, + Values: map[string]interface{}{ + "a": "b", + }, + }, + paths: []string{chartutil.ValuesfileName, "c.yaml"}, + want: map[string]interface{}{ + "a": "b", + "b": "d", + }, + }, + { + name: "unmarshal error", + chart: &helmchart.Chart{ + Files: []*helmchart.File{ + {Name: "invalid", Data: []byte("abcd")}, + }, + }, + paths: []string{"invalid"}, + wantErr: "unmarshaling values from 'invalid' failed", + }, + { + name: "error on invalid path", + chart: &helmchart.Chart{}, + paths: []string{"a.yaml"}, + wantErr: "no values file found at path 'a.yaml'", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + got, err := mergeChartValues(tt.chart, tt.paths) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(got).To(BeNil()) + return + } + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(Equal(tt.want)) + }) + } +} + +func Test_pathIsDir(t *testing.T) { + tests := []struct { + name string + p string + want bool + }{ + {name: "directory", p: "testdata/", want: true}, + {name: "file", p: "testdata/local-index.yaml", want: false}, + {name: "not found error", p: "testdata/does-not-exist.yaml", want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + g.Expect(pathIsDir(tt.p)).To(Equal(tt.want)) + }) + } +} diff --git a/internal/helm/chart_builder_test.go b/internal/helm/chart_builder_test.go index afc0107ce..a4252be8f 100644 --- a/internal/helm/chart_builder_test.go +++ b/internal/helm/chart_builder_test.go @@ -17,545 +17,27 @@ limitations under the License. package helm import ( - "context" "encoding/hex" - "fmt" "math/rand" "os" "path/filepath" - "sync" "testing" . "github.com/onsi/gomega" - helmchart "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" - "helm.sh/helm/v3/pkg/chartutil" - "helm.sh/helm/v3/pkg/repo" ) func TestChartBuildResult_String(t *testing.T) { g := NewWithT(t) - var result *ChartBuildResult + var result *ChartBuild g.Expect(result.String()).To(Equal("")) - result = &ChartBuildResult{} + result = &ChartBuild{} g.Expect(result.String()).To(Equal("")) - result = &ChartBuildResult{Path: "/foo/"} + result = &ChartBuild{Path: "/foo/"} g.Expect(result.String()).To(Equal("/foo/")) } -func TestChartBuilder_Build(t *testing.T) { - tests := []struct { - name string - baseDir string - path string - valueFiles []string - repositories map[string]*ChartRepository - getChartRepositoryCallback GetChartRepositoryCallback - wantErr string - }{ - { - name: "builds chart from directory", - path: "testdata/charts/helmchart", - }, - { - name: "builds chart from package", - path: "testdata/charts/helmchart-0.1.0.tgz", - }, - { - // TODO(hidde): add more diverse tests - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - g := NewWithT(t) - - b, err := NewChartBuilder(tt.path) - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(b).ToNot(BeNil()) - - b.WithBaseDir(tt.baseDir) - b.WithValueFiles(tt.valueFiles...) - b.WithChartRepositoryCallback(b.getChartRepositoryCallback) - for k, v := range tt.repositories { - b.WithChartRepository(k, v) - } - - out := tmpFile("build-0.1.0", ".tgz") - defer os.RemoveAll(out) - got, err := b.Build(context.TODO(), out) - if tt.wantErr != "" { - g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) - g.Expect(got).To(BeNil()) - return - } - - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(got).ToNot(BeNil()) - - g.Expect(got.Path).ToNot(BeEmpty()) - g.Expect(got.Path).To(Equal(out)) - g.Expect(got.Path).To(BeARegularFile()) - _, err = loader.Load(got.Path) - g.Expect(err).ToNot(HaveOccurred()) - }) - } -} - -func TestChartBuilder_load(t *testing.T) { - tests := []struct { - name string - path string - chart *helmchart.Chart - wantFunc func(g *WithT, c *helmchart.Chart) - wantErr string - }{ - { - name: "loads chart", - chart: nil, - path: "testdata/charts/helmchart-0.1.0.tgz", - wantFunc: func(g *WithT, c *helmchart.Chart) { - g.Expect(c.Metadata.Name).To(Equal("helmchart")) - g.Expect(c.Files).ToNot(BeZero()) - }, - }, - { - name: "overwrites chart without any files (metadata shim)", - chart: &helmchart.Chart{ - Metadata: &helmchart.Metadata{Name: "dummy"}, - }, - path: "testdata/charts/helmchart-0.1.0.tgz", - wantFunc: func(g *WithT, c *helmchart.Chart) { - g.Expect(c.Metadata.Name).To(Equal("helmchart")) - g.Expect(c.Files).ToNot(BeZero()) - }, - }, - { - name: "does not overwrite loaded chart", - chart: &helmchart.Chart{ - Metadata: &helmchart.Metadata{Name: "dummy"}, - Files: []*helmchart.File{ - {Name: "mock.yaml", Data: []byte("loaded chart")}, - }, - }, - path: "testdata/charts/helmchart-0.1.0.tgz", - wantFunc: func(g *WithT, c *helmchart.Chart) { - g.Expect(c.Metadata.Name).To(Equal("dummy")) - g.Expect(c.Files).To(HaveLen(1)) - }, - }, - { - name: "no path", - wantErr: "failed to load chart: path not set", - }, - { - name: "invalid chart", - path: "testdata/charts/empty.tgz", - wantErr: "failed to load chart: no files in chart archive", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - g := NewWithT(t) - - b := &ChartBuilder{ - path: tt.path, - chart: tt.chart, - } - err := b.load() - if tt.wantErr != "" { - g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) - return - } - - g.Expect(err).ToNot(HaveOccurred()) - if tt.wantFunc != nil { - tt.wantFunc(g, b.chart) - } - }) - } -} - -func TestChartBuilder_buildDependencies(t *testing.T) { - g := NewWithT(t) - - chartB, err := os.ReadFile("testdata/charts/helmchart-0.1.0.tgz") - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(chartB).ToNot(BeEmpty()) - - mockRepo := func() *ChartRepository { - return &ChartRepository{ - Client: &mockGetter{ - response: chartB, - }, - Index: &repo.IndexFile{ - Entries: map[string]repo.ChartVersions{ - "grafana": { - &repo.ChartVersion{ - Metadata: &helmchart.Metadata{ - Name: "grafana", - Version: "6.17.4", - }, - URLs: []string{"https://example.com/chart.tgz"}, - }, - }, - }, - }, - RWMutex: &sync.RWMutex{}, - } - } - - var mockCallback GetChartRepositoryCallback = func(url string) (*ChartRepository, error) { - if url == "https://grafana.github.io/helm-charts/" { - return mockRepo(), nil - } - return nil, fmt.Errorf("no repository for URL") - } - - tests := []struct { - name string - baseDir string - path string - chart *helmchart.Chart - fromDir bool - repositories map[string]*ChartRepository - getChartRepositoryCallback GetChartRepositoryCallback - wantCollectedDependencies int - wantErr string - }{ - { - name: "builds dependencies using callback", - fromDir: true, - baseDir: "testdata/charts", - path: "testdata/charts/helmchartwithdeps", - getChartRepositoryCallback: mockCallback, - wantCollectedDependencies: 2, - }, - { - name: "builds dependencies using repositories", - fromDir: true, - baseDir: "testdata/charts", - path: "testdata/charts/helmchartwithdeps", - repositories: map[string]*ChartRepository{ - "https://grafana.github.io/helm-charts/": mockRepo(), - }, - wantCollectedDependencies: 2, - }, - { - name: "skips dependency build for packaged chart", - path: "testdata/charts/helmchart-0.1.0.tgz", - }, - { - name: "attempts to load chart", - fromDir: true, - path: "testdata", - wantErr: "failed to ensure chart has no missing dependencies", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - g := NewWithT(t) - - b := &ChartBuilder{ - baseDir: tt.baseDir, - path: tt.path, - chart: tt.chart, - repositories: tt.repositories, - getChartRepositoryCallback: tt.getChartRepositoryCallback, - } - - result := &ChartBuildResult{SourceIsDir: tt.fromDir} - err := b.buildDependencies(context.TODO(), result) - if tt.wantErr != "" { - g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) - g.Expect(result.CollectedDependencies).To(BeZero()) - g.Expect(b.chart).To(Equal(tt.chart)) - return - } - - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(result).ToNot(BeNil()) - g.Expect(result.CollectedDependencies).To(Equal(tt.wantCollectedDependencies)) - if tt.wantCollectedDependencies > 0 { - g.Expect(b.chart).ToNot(Equal(tt.chart)) - } - }) - } -} - -func TestChartBuilder_mergeValues(t *testing.T) { - tests := []struct { - name string - baseDir string - path string - isDir bool - chart *helmchart.Chart - valueFiles []string - want map[string]interface{} - wantErr string - }{ - { - name: "merges chart values", - chart: &helmchart.Chart{ - Files: []*helmchart.File{ - {Name: "a.yaml", Data: []byte("a: b")}, - {Name: "b.yaml", Data: []byte("a: c")}, - }, - }, - valueFiles: []string{"a.yaml", "b.yaml"}, - want: map[string]interface{}{ - "a": "c", - }, - }, - { - name: "chart values merge error", - chart: &helmchart.Chart{ - Files: []*helmchart.File{ - {Name: "b.yaml", Data: []byte("a: c")}, - }, - }, - valueFiles: []string{"a.yaml"}, - wantErr: "failed to merge chart values", - }, - { - name: "merges file values", - isDir: true, - baseDir: "testdata/charts", - path: "helmchart", - valueFiles: []string{"helmchart/values-prod.yaml"}, - want: map[string]interface{}{ - "replicaCount": float64(2), - }, - }, - { - name: "file values merge error", - isDir: true, - baseDir: "testdata/charts", - path: "helmchart", - valueFiles: []string{"invalid.yaml"}, - wantErr: "failed to merge value files", - }, - { - name: "error on chart load failure", - baseDir: "testdata/charts", - path: "invalid", - wantErr: "failed to load chart", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - g := NewWithT(t) - - b := &ChartBuilder{ - baseDir: tt.baseDir, - path: tt.path, - chart: tt.chart, - valueFiles: tt.valueFiles, - } - - result := &ChartBuildResult{SourceIsDir: tt.isDir} - err := b.mergeValues(result) - if tt.wantErr != "" { - g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) - g.Expect(result.ValuesOverwrite).To(BeNil()) - return - } - - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(result.ValuesOverwrite).To(Equal(tt.want)) - }) - } -} - -func Test_mergeChartValues(t *testing.T) { - tests := []struct { - name string - chart *helmchart.Chart - paths []string - want map[string]interface{} - wantErr string - }{ - { - name: "merges values", - chart: &helmchart.Chart{ - Files: []*helmchart.File{ - {Name: "a.yaml", Data: []byte("a: b")}, - {Name: "b.yaml", Data: []byte("b: c")}, - {Name: "c.yaml", Data: []byte("b: d")}, - }, - }, - paths: []string{"a.yaml", "b.yaml", "c.yaml"}, - want: map[string]interface{}{ - "a": "b", - "b": "d", - }, - }, - { - name: "uses chart values", - chart: &helmchart.Chart{ - Files: []*helmchart.File{ - {Name: "c.yaml", Data: []byte("b: d")}, - }, - Values: map[string]interface{}{ - "a": "b", - }, - }, - paths: []string{chartutil.ValuesfileName, "c.yaml"}, - want: map[string]interface{}{ - "a": "b", - "b": "d", - }, - }, - { - name: "unmarshal error", - chart: &helmchart.Chart{ - Files: []*helmchart.File{ - {Name: "invalid", Data: []byte("abcd")}, - }, - }, - paths: []string{"invalid"}, - wantErr: "unmarshaling values from 'invalid' failed", - }, - { - name: "error on invalid path", - chart: &helmchart.Chart{}, - paths: []string{"a.yaml"}, - wantErr: "no values file found at path 'a.yaml'", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - g := NewWithT(t) - - got, err := mergeChartValues(tt.chart, tt.paths) - if tt.wantErr != "" { - g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) - g.Expect(got).To(BeNil()) - return - } - - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(got).To(Equal(tt.want)) - }) - } -} - -func Test_mergeFileValues(t *testing.T) { - tests := []struct { - name string - files []*helmchart.File - paths []string - want map[string]interface{} - wantErr string - }{ - { - name: "merges values from files", - files: []*helmchart.File{ - {Name: "a.yaml", Data: []byte("a: b")}, - {Name: "b.yaml", Data: []byte("b: c")}, - {Name: "c.yaml", Data: []byte("b: d")}, - }, - paths: []string{"a.yaml", "b.yaml", "c.yaml"}, - want: map[string]interface{}{ - "a": "b", - "b": "d", - }, - }, - { - name: "illegal traverse", - paths: []string{"../../../traversing/illegally/a/p/a/b"}, - wantErr: "no values file found at path '/traversing/illegally/a/p/a/b'", - }, - { - name: "unmarshal error", - files: []*helmchart.File{ - {Name: "invalid", Data: []byte("abcd")}, - }, - paths: []string{"invalid"}, - wantErr: "unmarshaling values from 'invalid' failed", - }, - { - name: "error on invalid path", - paths: []string{"a.yaml"}, - wantErr: "no values file found at path '/a.yaml'", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - g := NewWithT(t) - - baseDir, err := os.MkdirTemp("", "merge-file-values-*") - g.Expect(err).ToNot(HaveOccurred()) - defer os.RemoveAll(baseDir) - - for _, f := range tt.files { - g.Expect(os.WriteFile(filepath.Join(baseDir, f.Name), f.Data, 0644)).To(Succeed()) - } - - got, err := mergeFileValues(baseDir, tt.paths) - if tt.wantErr != "" { - g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) - g.Expect(got).To(BeNil()) - return - } - - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(got).To(Equal(tt.want)) - }) - } -} - -func Test_copyFileToPath(t *testing.T) { - tests := []struct { - name string - in string - wantErr string - }{ - { - name: "copies input file", - in: "testdata/local-index.yaml", - }, - { - name: "invalid input file", - in: "testdata/invalid.tgz", - wantErr: "failed to open file to copy from", - }, - { - name: "invalid input directory", - in: "testdata/charts", - wantErr: "failed to read from source during copy", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - g := NewWithT(t) - - out := tmpFile("copy-0.1.0", ".tgz") - defer os.RemoveAll(out) - err := copyFileToPath(tt.in, out) - if tt.wantErr != "" { - g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) - return - } - - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(out).To(BeARegularFile()) - f1, err := os.ReadFile(tt.in) - g.Expect(err).ToNot(HaveOccurred()) - f2, err := os.ReadFile(out) - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(f2).To(Equal(f1)) - }) - } -} - func Test_packageToPath(t *testing.T) { g := NewWithT(t) @@ -572,25 +54,6 @@ func Test_packageToPath(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) } -func Test_pathIsDir(t *testing.T) { - tests := []struct { - name string - p string - want bool - }{ - {name: "directory", p: "testdata/", want: true}, - {name: "file", p: "testdata/local-index.yaml", want: false}, - {name: "not found error", p: "testdata/does-not-exist.yaml", want: false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - g := NewWithT(t) - - g.Expect(pathIsDir(tt.p)).To(Equal(tt.want)) - }) - } -} - func tmpFile(prefix, suffix string) string { randBytes := make([]byte, 16) rand.Read(randBytes) diff --git a/internal/helm/chart_test.go b/internal/helm/chart_test.go index 23d50b96b..ac7114e87 100644 --- a/internal/helm/chart_test.go +++ b/internal/helm/chart_test.go @@ -25,8 +25,9 @@ import ( ) var ( - originalValuesFixture = []byte("override: original") - chartFilesFixture = []*helmchart.File{ + originalValuesFixture = []byte(`override: original +`) + chartFilesFixture = []*helmchart.File{ { Name: "values.yaml", Data: originalValuesFixture, @@ -69,19 +70,14 @@ func TestOverwriteChartDefaultValues(t *testing.T) { desc: "valid override", chart: chartFixture, ok: true, - data: []byte("override: test"), + data: []byte(`override: test +`), }, { desc: "empty override", chart: chartFixture, ok: true, - data: []byte(""), - }, - { - desc: "invalid", - chart: chartFixture, - data: []byte("!fail:"), - expectErr: true, + data: []byte(``), }, } for _, tt := range testCases { @@ -89,7 +85,9 @@ func TestOverwriteChartDefaultValues(t *testing.T) { g := NewWithT(t) fixture := tt.chart - ok, err := OverwriteChartDefaultValues(&fixture, tt.data) + vals, err := chartutil.ReadValues(tt.data) + g.Expect(err).ToNot(HaveOccurred()) + ok, err := OverwriteChartDefaultValues(&fixture, vals) g.Expect(ok).To(Equal(tt.ok)) if tt.expectErr { diff --git a/internal/helm/dependency_manager.go b/internal/helm/dependency_manager.go index 043b0e7e3..b8cd78571 100644 --- a/internal/helm/dependency_manager.go +++ b/internal/helm/dependency_manager.go @@ -37,72 +37,77 @@ import ( // or an error describing why it could not be returned. type GetChartRepositoryCallback func(url string) (*ChartRepository, error) -// DependencyManager manages dependencies for a Helm chart, downloading -// only those that are missing from the chart it holds. +// DependencyManager manages dependencies for a Helm chart. type DependencyManager struct { - // chart contains the chart.Chart from the path. - chart *helmchart.Chart - - // baseDir is the chroot path for dependency manager operations, - // Dependencies that hold a local (relative) path reference are not - // allowed to traverse outside this directory. - baseDir string - - // path is the path of the chart relative to the baseDir, - // the combination of the baseDir and path is used to - // determine the absolute path of a local dependency. - path string - // repositories contains a map of ChartRepository indexed by their // normalized URL. It is used as a lookup table for missing // dependencies. repositories map[string]*ChartRepository - // getChartRepositoryCallback can be set to an on-demand get - // callback which returned result is cached to repositories. - getChartRepositoryCallback GetChartRepositoryCallback + // getRepositoryCallback can be set to an on-demand GetChartRepositoryCallback + // which returned result is cached to repositories. + getRepositoryCallback GetChartRepositoryCallback - // workers is the number of concurrent chart-add operations during + // concurrent is the number of concurrent chart-add operations during // Build. Defaults to 1 (non-concurrent). - workers int64 + concurrent int64 // mu contains the lock for chart writes. mu sync.Mutex } -func NewDependencyManager(chart *helmchart.Chart, baseDir, path string) *DependencyManager { - return &DependencyManager{ - chart: chart, - baseDir: baseDir, - path: path, - } +type DependencyManagerOption interface { + applyToDependencyManager(dm *DependencyManager) } -func (dm *DependencyManager) WithRepositories(r map[string]*ChartRepository) *DependencyManager { - dm.repositories = r - return dm +type WithRepositories map[string]*ChartRepository + +func (o WithRepositories) applyToDependencyManager(dm *DependencyManager) { + dm.repositories = o } -func (dm *DependencyManager) WithChartRepositoryCallback(c GetChartRepositoryCallback) *DependencyManager { - dm.getChartRepositoryCallback = c - return dm +type WithRepositoryCallback GetChartRepositoryCallback + +func (o WithRepositoryCallback) applyToDependencyManager(dm *DependencyManager) { + dm.getRepositoryCallback = GetChartRepositoryCallback(o) +} + +type WithConcurrent int64 + +func (o WithConcurrent) applyToDependencyManager(dm *DependencyManager) { + dm.concurrent = int64(o) } -func (dm *DependencyManager) WithWorkers(w int64) *DependencyManager { - dm.workers = w +// NewDependencyManager returns a new DependencyManager configured with the given +// DependencyManagerOption list. +func NewDependencyManager(opts ...DependencyManagerOption) *DependencyManager { + dm := &DependencyManager{} + for _, v := range opts { + v.applyToDependencyManager(dm) + } return dm } -// Build compiles and builds the dependencies of the chart with the -// configured number of workers. -func (dm *DependencyManager) Build(ctx context.Context) (int, error) { +func (dm *DependencyManager) Clear() []error { + var errs []error + for _, v := range dm.repositories { + v.Unload() + errs = append(errs, v.RemoveCache()) + } + return errs +} + +// Build compiles a set of missing dependencies from chart.Chart, and attempts to +// resolve and build them using the information from ChartReference. +// It returns the number of resolved local and remote dependencies, or an error. +func (dm *DependencyManager) Build(ctx context.Context, ref ChartReference, chart *helmchart.Chart) (int, error) { // Collect dependency metadata var ( - deps = dm.chart.Dependencies() - reqs = dm.chart.Metadata.Dependencies + deps = chart.Dependencies() + reqs = chart.Metadata.Dependencies ) // Lock file takes precedence - if lock := dm.chart.Lock; lock != nil { + if lock := chart.Lock; lock != nil { reqs = lock.Dependencies } @@ -113,31 +118,32 @@ func (dm *DependencyManager) Build(ctx context.Context) (int, error) { } // Run the build for the missing dependencies - if err := dm.build(ctx, missing); err != nil { + if err := dm.build(ctx, ref, chart, missing); err != nil { return 0, err } return len(missing), nil } -// build (concurrently) adds the given list of deps to the chart with the configured -// number of workers. It returns the first error, cancelling all other workers. -func (dm *DependencyManager) build(ctx context.Context, deps map[string]*helmchart.Dependency) error { - workers := dm.workers - if workers <= 0 { - workers = 1 - } +// chartWithLock holds a chart.Chart with a sync.Mutex to lock for writes. +type chartWithLock struct { + *helmchart.Chart + mu sync.Mutex +} - // Garbage collect temporary cached ChartRepository indexes - defer func() { - for _, v := range dm.repositories { - v.Unload() - _ = v.RemoveCache() - } - }() +// build adds the given list of deps to the chart with the configured number of +// concurrent workers. If the chart.Chart references a local dependency but no +// LocalChartReference is given, or any dependency could not be added, an error +// is returned. The first error it encounters cancels all other workers. +func (dm *DependencyManager) build(ctx context.Context, ref ChartReference, chart *helmchart.Chart, deps map[string]*helmchart.Dependency) error { + current := dm.concurrent + if current <= 0 { + current = 1 + } group, groupCtx := errgroup.WithContext(ctx) group.Go(func() error { - sem := semaphore.NewWeighted(workers) + sem := semaphore.NewWeighted(current) + chart := &chartWithLock{Chart: chart} for name, dep := range deps { name, dep := name, dep if err := sem.Acquire(groupCtx, 1); err != nil { @@ -146,12 +152,17 @@ func (dm *DependencyManager) build(ctx context.Context, deps map[string]*helmcha group.Go(func() (err error) { defer sem.Release(1) if isLocalDep(dep) { - if err = dm.addLocalDependency(dep); err != nil { + localRef, ok := ref.(LocalChartReference) + if !ok { + err = fmt.Errorf("failed to add local dependency '%s': no local chart reference", name) + return + } + if err = dm.addLocalDependency(localRef, chart, dep); err != nil { err = fmt.Errorf("failed to add local dependency '%s': %w", name, err) } return } - if err = dm.addRemoteDependency(dep); err != nil { + if err = dm.addRemoteDependency(chart, dep); err != nil { err = fmt.Errorf("failed to add remote dependency '%s': %w", name, err) } return @@ -162,17 +173,17 @@ func (dm *DependencyManager) build(ctx context.Context, deps map[string]*helmcha return group.Wait() } -// addLocalDependency attempts to resolve and add the given local chart.Dependency to the chart. -func (dm *DependencyManager) addLocalDependency(dep *helmchart.Dependency) error { - sLocalChartPath, err := dm.secureLocalChartPath(dep) +// addLocalDependency attempts to resolve and add the given local chart.Dependency +// to the chart. +func (dm *DependencyManager) addLocalDependency(ref LocalChartReference, chart *chartWithLock, dep *helmchart.Dependency) error { + sLocalChartPath, err := dm.secureLocalChartPath(ref, dep) if err != nil { return err } if _, err := os.Stat(sLocalChartPath); err != nil { if os.IsNotExist(err) { - return fmt.Errorf("no chart found at '%s' (reference '%s')", - strings.TrimPrefix(sLocalChartPath, dm.baseDir), dep.Repository) + return fmt.Errorf("no chart found at '%s' (reference '%s')", sLocalChartPath, dep.Repository) } return err } @@ -186,7 +197,7 @@ func (dm *DependencyManager) addLocalDependency(dep *helmchart.Dependency) error ch, err := loader.Load(sLocalChartPath) if err != nil { return fmt.Errorf("failed to load chart from '%s' (reference '%s'): %w", - strings.TrimPrefix(sLocalChartPath, dm.baseDir), dep.Repository, err) + strings.TrimPrefix(sLocalChartPath, ref.BaseDir), dep.Repository, err) } ver, err := semver.NewVersion(ch.Metadata.Version) @@ -199,14 +210,16 @@ func (dm *DependencyManager) addLocalDependency(dep *helmchart.Dependency) error return err } - dm.mu.Lock() - dm.chart.AddDependency(ch) - dm.mu.Unlock() + chart.mu.Lock() + chart.AddDependency(ch) + chart.mu.Unlock() return nil } -// addRemoteDependency attempts to resolve and add the given remote chart.Dependency to the chart. -func (dm *DependencyManager) addRemoteDependency(dep *helmchart.Dependency) error { +// addRemoteDependency attempts to resolve and add the given remote chart.Dependency +// to the chart. It locks the chartWithLock before the downloaded dependency is +// added to the chart. +func (dm *DependencyManager) addRemoteDependency(chart *chartWithLock, dep *helmchart.Dependency) error { repo, err := dm.resolveRepository(dep.Repository) if err != nil { return err @@ -216,7 +229,6 @@ func (dm *DependencyManager) addRemoteDependency(dep *helmchart.Dependency) erro return fmt.Errorf("failed to load index for '%s': %w", dep.Name, err) } - ver, err := repo.Get(dep.Name, dep.Version) if err != nil { return err @@ -230,28 +242,28 @@ func (dm *DependencyManager) addRemoteDependency(dep *helmchart.Dependency) erro return fmt.Errorf("failed to load downloaded archive of version '%s': %w", ver.Version, err) } - dm.mu.Lock() - dm.chart.AddDependency(ch) - dm.mu.Unlock() + chart.mu.Lock() + chart.AddDependency(ch) + chart.mu.Unlock() return nil } // resolveRepository first attempts to resolve the url from the repositories, falling back -// to getChartRepositoryCallback if set. It returns the resolved ChartRepository, or an error. +// to getRepositoryCallback if set. It returns the resolved ChartRepository, or an error. func (dm *DependencyManager) resolveRepository(url string) (_ *ChartRepository, err error) { dm.mu.Lock() defer dm.mu.Unlock() nUrl := NormalizeChartRepositoryURL(url) if _, ok := dm.repositories[nUrl]; !ok { - if dm.getChartRepositoryCallback == nil { + if dm.getRepositoryCallback == nil { err = fmt.Errorf("no chart repository for URL '%s'", nUrl) return } if dm.repositories == nil { dm.repositories = map[string]*ChartRepository{} } - if dm.repositories[nUrl], err = dm.getChartRepositoryCallback(nUrl); err != nil { + if dm.repositories[nUrl], err = dm.getRepositoryCallback(nUrl); err != nil { err = fmt.Errorf("failed to get chart repository for URL '%s': %w", nUrl, err) return } @@ -260,8 +272,9 @@ func (dm *DependencyManager) resolveRepository(url string) (_ *ChartRepository, } // secureLocalChartPath returns the secure absolute path of a local dependency. -// It does not allow the dependency's path to be outside the scope of baseDir. -func (dm *DependencyManager) secureLocalChartPath(dep *helmchart.Dependency) (string, error) { +// It does not allow the dependency's path to be outside the scope of +// LocalChartReference.BaseDir. +func (dm *DependencyManager) secureLocalChartPath(ref LocalChartReference, dep *helmchart.Dependency) (string, error) { localUrl, err := url.Parse(dep.Repository) if err != nil { return "", fmt.Errorf("failed to parse alleged local chart reference: %w", err) @@ -269,7 +282,11 @@ func (dm *DependencyManager) secureLocalChartPath(dep *helmchart.Dependency) (st if localUrl.Scheme != "" && localUrl.Scheme != "file" { return "", fmt.Errorf("'%s' is not a local chart reference", dep.Repository) } - return securejoin.SecureJoin(dm.baseDir, filepath.Join(dm.path, localUrl.Host, localUrl.Path)) + relPath, err := filepath.Rel(ref.BaseDir, ref.Path) + if err != nil { + return "", err + } + return securejoin.SecureJoin(ref.BaseDir, filepath.Join(relPath, localUrl.Host, localUrl.Path)) } // collectMissing returns a map with reqs that are missing from current, diff --git a/internal/helm/dependency_manager_test.go b/internal/helm/dependency_manager_test.go index e51e6b768..388eff1f4 100644 --- a/internal/helm/dependency_manager_test.go +++ b/internal/helm/dependency_manager_test.go @@ -88,10 +88,10 @@ func TestDependencyManager_Build(t *testing.T) { chart, err := loader.Load(filepath.Join(tt.baseDir, tt.path)) g.Expect(err).ToNot(HaveOccurred()) - got, err := NewDependencyManager(chart, tt.baseDir, tt.path). - WithRepositories(tt.repositories). - WithChartRepositoryCallback(tt.getChartRepositoryCallback). - Build(context.TODO()) + got, err := NewDependencyManager( + WithRepositories(tt.repositories), + WithRepositoryCallback(tt.getChartRepositoryCallback), + ).Build(context.TODO(), LocalChartReference{BaseDir: tt.baseDir, Path: tt.path}, chart) if tt.wantErr != "" { g.Expect(err).To(HaveOccurred()) @@ -134,10 +134,8 @@ func TestDependencyManager_build(t *testing.T) { t.Run(tt.name, func(t *testing.T) { g := NewWithT(t) - dm := &DependencyManager{ - baseDir: "testdata/charts", - } - err := dm.build(context.TODO(), tt.deps) + dm := NewDependencyManager() + err := dm.build(context.TODO(), LocalChartReference{}, &helmchart.Chart{}, tt.deps) if tt.wantErr != "" { g.Expect(err).To(HaveOccurred()) return @@ -182,7 +180,7 @@ func TestDependencyManager_addLocalDependency(t *testing.T) { Version: chartVersion, Repository: "file://../../../absolutely/invalid", }, - wantErr: "no chart found at 'absolutely/invalid'", + wantErr: "no chart found at 'testdata/charts/absolutely/invalid'", }, { name: "invalid chart archive", @@ -191,7 +189,7 @@ func TestDependencyManager_addLocalDependency(t *testing.T) { Version: chartVersion, Repository: "file://../empty.tgz", }, - wantErr: "failed to load chart from 'empty.tgz'", + wantErr: "failed to load chart from '/empty.tgz'", }, { name: "invalid constraint", @@ -207,13 +205,10 @@ func TestDependencyManager_addLocalDependency(t *testing.T) { t.Run(tt.name, func(t *testing.T) { g := NewWithT(t) - dm := &DependencyManager{ - chart: &helmchart.Chart{}, - baseDir: "testdata/charts/", - path: "helmchartwithdeps", - } - - err := dm.addLocalDependency(tt.dep) + dm := NewDependencyManager() + chart := &helmchart.Chart{} + err := dm.addLocalDependency(LocalChartReference{BaseDir: "testdata/charts", Path: "helmchartwithdeps"}, + &chartWithLock{Chart: chart}, tt.dep) if tt.wantErr != "" { g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) @@ -389,10 +384,10 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) { g := NewWithT(t) dm := &DependencyManager{ - chart: &helmchart.Chart{}, repositories: tt.repositories, } - err := dm.addRemoteDependency(tt.dep) + chart := &helmchart.Chart{} + err := dm.addRemoteDependency(&chartWithLock{Chart: chart}, tt.dep) if tt.wantErr != "" { g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) @@ -400,7 +395,7 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) { } g.Expect(err).ToNot(HaveOccurred()) if tt.wantFunc != nil { - tt.wantFunc(g, dm.chart) + tt.wantFunc(g, chart) } }) } @@ -455,8 +450,8 @@ func TestDependencyManager_resolveRepository(t *testing.T) { g := NewWithT(t) dm := &DependencyManager{ - repositories: tt.repositories, - getChartRepositoryCallback: tt.getChartRepositoryCallback, + repositories: tt.repositories, + getRepositoryCallback: tt.getChartRepositoryCallback, } got, err := dm.resolveRepository(tt.url) @@ -522,11 +517,8 @@ func TestDependencyManager_secureLocalChartPath(t *testing.T) { t.Run(tt.name, func(t *testing.T) { g := NewWithT(t) - dm := &DependencyManager{ - baseDir: tt.baseDir, - path: tt.path, - } - got, err := dm.secureLocalChartPath(tt.dep) + dm := NewDependencyManager() + got, err := dm.secureLocalChartPath(LocalChartReference{BaseDir: tt.baseDir, Path: tt.path}, tt.dep) if tt.wantErr != "" { g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) diff --git a/internal/helm/getter.go b/internal/helm/getter.go index b0f07e96b..1ca8b0e9b 100644 --- a/internal/helm/getter.go +++ b/internal/helm/getter.go @@ -19,31 +19,30 @@ package helm import ( "fmt" "os" - "path/filepath" "helm.sh/helm/v3/pkg/getter" corev1 "k8s.io/api/core/v1" ) // ClientOptionsFromSecret constructs a getter.Option slice for the given secret. -// It returns the slice, and a callback to remove temporary files. -func ClientOptionsFromSecret(secret corev1.Secret) ([]getter.Option, func(), error) { +// It returns the slice, or an error. +func ClientOptionsFromSecret(dir string, secret corev1.Secret) ([]getter.Option, error) { var opts []getter.Option basicAuth, err := BasicAuthFromSecret(secret) if err != nil { - return opts, nil, err + return opts, err } if basicAuth != nil { opts = append(opts, basicAuth) } - tlsClientConfig, cleanup, err := TLSClientConfigFromSecret(secret) + tlsClientConfig, err := TLSClientConfigFromSecret(dir, secret) if err != nil { - return opts, nil, err + return opts, err } if tlsClientConfig != nil { opts = append(opts, tlsClientConfig) } - return opts, cleanup, nil + return opts, nil } // BasicAuthFromSecret attempts to construct a basic auth getter.Option for the @@ -63,50 +62,65 @@ func BasicAuthFromSecret(secret corev1.Secret) (getter.Option, error) { } // TLSClientConfigFromSecret attempts to construct a TLS client config -// getter.Option for the given v1.Secret. It returns the getter.Option and a -// callback to remove the temporary TLS files. +// getter.Option for the given v1.Secret, placing the required TLS config +// related files in the given directory. It returns the getter.Option, or +// an error. // // Secrets with no certFile, keyFile, AND caFile are ignored, if only a // certBytes OR keyBytes is defined it returns an error. -func TLSClientConfigFromSecret(secret corev1.Secret) (getter.Option, func(), error) { +func TLSClientConfigFromSecret(dir string, secret corev1.Secret) (getter.Option, error) { certBytes, keyBytes, caBytes := secret.Data["certFile"], secret.Data["keyFile"], secret.Data["caFile"] switch { case len(certBytes)+len(keyBytes)+len(caBytes) == 0: - return nil, func() {}, nil + return nil, nil case (len(certBytes) > 0 && len(keyBytes) == 0) || (len(keyBytes) > 0 && len(certBytes) == 0): - return nil, nil, fmt.Errorf("invalid '%s' secret data: fields 'certFile' and 'keyFile' require each other's presence", + return nil, fmt.Errorf("invalid '%s' secret data: fields 'certFile' and 'keyFile' require each other's presence", secret.Name) } - // create tmp dir for TLS files - tmp, err := os.MkdirTemp("", "helm-tls-"+secret.Name) - if err != nil { - return nil, nil, err - } - cleanup := func() { os.RemoveAll(tmp) } - - var certFile, keyFile, caFile string - + var certPath, keyPath, caPath string if len(certBytes) > 0 && len(keyBytes) > 0 { - certFile = filepath.Join(tmp, "cert.crt") - if err := os.WriteFile(certFile, certBytes, 0644); err != nil { - cleanup() - return nil, nil, err + certFile, err := os.CreateTemp(dir, "cert-*.crt") + if err != nil { + return nil, err + } + if _, err = certFile.Write(certBytes); err != nil { + _ = certFile.Close() + return nil, err } - keyFile = filepath.Join(tmp, "key.crt") - if err := os.WriteFile(keyFile, keyBytes, 0644); err != nil { - cleanup() - return nil, nil, err + if err = certFile.Close(); err != nil { + return nil, err } + certPath = certFile.Name() + + keyFile, err := os.CreateTemp(dir, "key-*.crt") + if err != nil { + return nil, err + } + if _, err = keyFile.Write(keyBytes); err != nil { + _ = keyFile.Close() + return nil, err + } + if err = keyFile.Close(); err != nil { + return nil, err + } + keyPath = keyFile.Name() } if len(caBytes) > 0 { - caFile = filepath.Join(tmp, "ca.pem") - if err := os.WriteFile(caFile, caBytes, 0644); err != nil { - cleanup() - return nil, nil, err + caFile, err := os.CreateTemp(dir, "ca-*.pem") + if err != nil { + return nil, err + } + if _, err = caFile.Write(caBytes); err != nil { + _ = caFile.Close() + return nil, err + } + if err = caFile.Close(); err != nil { + return nil, err } + caPath = caFile.Name() } - return getter.WithTLSClientConfig(certFile, keyFile, caFile), cleanup, nil + return getter.WithTLSClientConfig(certPath, keyPath, caPath), nil } diff --git a/internal/helm/getter_test.go b/internal/helm/getter_test.go index bd4e1058c..2c55e7cbb 100644 --- a/internal/helm/getter_test.go +++ b/internal/helm/getter_test.go @@ -17,6 +17,7 @@ limitations under the License. package helm import ( + "os" "testing" corev1 "k8s.io/api/core/v1" @@ -56,10 +57,14 @@ func TestClientOptionsFromSecret(t *testing.T) { secret.Data[k] = v } } - got, cleanup, err := ClientOptionsFromSecret(secret) - if cleanup != nil { - defer cleanup() + + tmpDir, err := os.MkdirTemp("", "client-opts-secret-") + if err != nil { + t.Fatal(err) } + defer os.RemoveAll(tmpDir) + + got, err := ClientOptionsFromSecret(tmpDir, secret) if err != nil { t.Errorf("ClientOptionsFromSecret() error = %v", err) return @@ -123,10 +128,14 @@ func TestTLSClientConfigFromSecret(t *testing.T) { if tt.modify != nil { tt.modify(secret) } - got, cleanup, err := TLSClientConfigFromSecret(*secret) - if cleanup != nil { - defer cleanup() + + tmpDir, err := os.MkdirTemp("", "client-opts-secret-") + if err != nil { + t.Fatal(err) } + defer os.RemoveAll(tmpDir) + + got, err := TLSClientConfigFromSecret(tmpDir, *secret) if (err != nil) != tt.wantErr { t.Errorf("TLSClientConfigFromSecret() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/internal/helm/helm.go b/internal/helm/helm.go new file mode 100644 index 000000000..ec9668542 --- /dev/null +++ b/internal/helm/helm.go @@ -0,0 +1,29 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helm + +// This list defines a set of global variables used to ensure Helm files loaded +// into memory during runtime do not exceed defined upper bound limits. +var ( + // MaxIndexSize is the max allowed file size in bytes of a ChartRepository. + MaxIndexSize int64 = 50 << 20 + // MaxChartSize is the max allowed file size in bytes of a Helm Chart. + MaxChartSize int64 = 2 << 20 + // MaxChartFileSize is the max allowed file size in bytes of any arbitrary + // file originating from a chart. + MaxChartFileSize int64 = 2 << 10 +) diff --git a/internal/helm/repository.go b/internal/helm/repository.go index e2446f944..eb9e668a1 100644 --- a/internal/helm/repository.go +++ b/internal/helm/repository.go @@ -234,6 +234,16 @@ func (r *ChartRepository) LoadIndexFromBytes(b []byte) error { // LoadFromFile reads the file at the given path and loads it into Index. func (r *ChartRepository) LoadFromFile(path string) error { + stat, err := os.Stat(path) + if err != nil || stat.IsDir() { + if err == nil { + err = fmt.Errorf("'%s' is a directory", path) + } + return err + } + if stat.Size() > MaxIndexSize { + return fmt.Errorf("size of index '%s' exceeds '%d' limit", stat.Name(), MaxIndexSize) + } b, err := os.ReadFile(path) if err != nil { return err @@ -342,7 +352,7 @@ func (r *ChartRepository) HasCacheFile() bool { // Unload can be used to signal the Go garbage collector the Index can // be freed from memory if the ChartRepository object is expected to // continue to exist in the stack for some time. -func (r *ChartRepository) Unload() { +func (r *ChartRepository) Unload() { if r == nil { return } diff --git a/internal/helm/repository_test.go b/internal/helm/repository_test.go index 0d2077dfd..9c124b791 100644 --- a/internal/helm/repository_test.go +++ b/internal/helm/repository_test.go @@ -416,7 +416,7 @@ func TestChartRepository_LoadFromCache(t *testing.T) { { name: "invalid cache path", cachePath: "invalid", - wantErr: "open invalid: no such file", + wantErr: "stat invalid: no such file", }, { name: "no cache path", From 9abbdd80a6eda9d47a7632328237e16def550a1f Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Fri, 5 Nov 2021 15:29:40 +0100 Subject: [PATCH 07/23] controllers: rough wiring of Helm chart builder This commit starts wiring the factored out Helm chart build logic into the reconciler to ensure, validating the API capabilities. Signed-off-by: Hidde Beydals --- controllers/helmchart_controller.go | 509 +++++++---------------- controllers/helmchart_controller_test.go | 1 + controllers/helmrepository_controller.go | 11 +- 3 files changed, 160 insertions(+), 361 deletions(-) diff --git a/controllers/helmchart_controller.go b/controllers/helmchart_controller.go index 5d4f952cd..bcb8f8e79 100644 --- a/controllers/helmchart_controller.go +++ b/controllers/helmchart_controller.go @@ -19,7 +19,6 @@ package controllers import ( "context" "fmt" - "io" "net/url" "os" "path/filepath" @@ -27,14 +26,11 @@ import ( "strings" "time" - "github.com/Masterminds/semver/v3" securejoin "github.com/cyphar/filepath-securejoin" "github.com/go-logr/logr" - helmchart "helm.sh/helm/v3/pkg/chart" - "helm.sh/helm/v3/pkg/chart/loader" - "helm.sh/helm/v3/pkg/chartutil" "helm.sh/helm/v3/pkg/getter" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -50,13 +46,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - "sigs.k8s.io/yaml" "github.com/fluxcd/pkg/apis/meta" "github.com/fluxcd/pkg/runtime/events" "github.com/fluxcd/pkg/runtime/metrics" "github.com/fluxcd/pkg/runtime/predicates" - "github.com/fluxcd/pkg/runtime/transform" "github.com/fluxcd/pkg/untar" sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" @@ -202,6 +196,19 @@ func (r *HelmChartReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{Requeue: true}, err } + // Create working directory + workDir, err := os.MkdirTemp("", chart.Kind + "-" + chart.Namespace + "-" + chart.Name + "-") + if err != nil { + err = fmt.Errorf("failed to create temporary working directory: %w", err) + chart = sourcev1.HelmChartNotReady(*chart.DeepCopy(), sourcev1.ChartPullFailedReason, err.Error()) + if err := r.updateStatus(ctx, req, chart.Status); err != nil { + log.Error(err, "unable to update status") + } + r.recordReadiness(ctx, chart) + return ctrl.Result{Requeue: true}, err + } + defer os.RemoveAll(workDir) + // Perform the reconciliation for the chart source type var reconciledChart sourcev1.HelmChart var reconcileErr error @@ -222,10 +229,10 @@ func (r *HelmChartReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( // Do not requeue as there is no chance on recovery. return ctrl.Result{Requeue: false}, nil } - reconciledChart, reconcileErr = r.reconcileFromHelmRepository(ctx, *typedSource, *chart.DeepCopy(), changed) + reconciledChart, reconcileErr = r.fromHelmRepository(ctx, *typedSource, *chart.DeepCopy(), workDir, changed) case *sourcev1.GitRepository, *sourcev1.Bucket: - reconciledChart, reconcileErr = r.reconcileFromTarballArtifact(ctx, *typedSource.GetArtifact(), - *chart.DeepCopy(), changed) + reconciledChart, reconcileErr = r.fromTarballArtifact(ctx, *typedSource.GetArtifact(), *chart.DeepCopy(), + workDir, changed) default: err := fmt.Errorf("unable to reconcile unsupported source reference kind '%s'", chart.Spec.SourceRef.Kind) return ctrl.Result{Requeue: false}, err @@ -297,8 +304,8 @@ func (r *HelmChartReconciler) getSource(ctx context.Context, chart sourcev1.Helm return source, nil } -func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context, - repository sourcev1.HelmRepository, chart sourcev1.HelmChart, force bool) (sourcev1.HelmChart, error) { +func (r *HelmChartReconciler) fromHelmRepository(ctx context.Context, repository sourcev1.HelmRepository, + chart sourcev1.HelmChart, workDir string, force bool) (sourcev1.HelmChart, error) { // Configure ChartRepository getter options clientOpts := []getter.Option{ getter.WithURL(repository.Spec.URL), @@ -308,17 +315,21 @@ func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context, if secret, err := r.getHelmRepositorySecret(ctx, &repository); err != nil { return sourcev1.HelmChartNotReady(chart, sourcev1.AuthenticationFailedReason, err.Error()), err } else if secret != nil { - opts, cleanup, err := helm.ClientOptionsFromSecret(*secret) + // Create temporary working directory for credentials + authDir := filepath.Join(workDir, "creds") + if err := os.Mkdir(authDir, 0700); err != nil { + err = fmt.Errorf("failed to create temporary directory for repository credentials: %w", err) + } + opts, err := helm.ClientOptionsFromSecret(authDir, *secret) if err != nil { - err = fmt.Errorf("auth options error: %w", err) + err = fmt.Errorf("failed to create client options for HelmRepository '%s': %w", repository.Name, err) return sourcev1.HelmChartNotReady(chart, sourcev1.AuthenticationFailedReason, err.Error()), err } - defer cleanup() clientOpts = append(clientOpts, opts...) } - // Initialize the chart repository and load the index file - chartRepo, err := helm.NewChartRepository(repository.Spec.URL, r.Getters, clientOpts) + // Initialize the chart repository + chartRepo, err := helm.NewChartRepository(repository.Spec.URL, r.Storage.LocalPath(*repository.GetArtifact()), r.Getters, clientOpts) if err != nil { switch err.(type) { case *url.Error: @@ -327,29 +338,33 @@ func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context, return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err } } - indexFile, err := os.Open(r.Storage.LocalPath(*repository.GetArtifact())) - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - b, err := io.ReadAll(indexFile) - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err - } - if err = chartRepo.LoadIndex(b); err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err + + var cachedChart string + if artifact := chart.GetArtifact(); artifact != nil { + cachedChart = artifact.Path } - // Lookup the chart version in the chart repository index - chartVer, err := chartRepo.Get(chart.Spec.Chart, chart.Spec.Version) + // Build the chart + cBuilder := helm.NewRemoteChartBuilder(chartRepo) + ref := helm.RemoteChartReference{Name: chart.Spec.Chart, Version: chart.Spec.Version} + opts := helm.BuildOptions{ + ValueFiles: chart.GetValuesFiles(), + CachedChart: cachedChart, + Force: force, + } + build, err := cBuilder.Build(ctx, ref, filepath.Join(workDir, "chart.tgz"), opts) if err != nil { return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err } - // Return early if the revision is still the same as the current artifact - newArtifact := r.Storage.NewArtifactFor(chart.Kind, chart.GetObjectMeta(), chartVer.Version, - fmt.Sprintf("%s-%s.tgz", chartVer.Name, chartVer.Version)) - if !force && repository.GetArtifact().HasRevision(newArtifact.Revision) { - if newArtifact.URL != chart.GetArtifact().URL { + newArtifact := r.Storage.NewArtifactFor(chart.Kind, chart.GetObjectMeta(), build.Version, + fmt.Sprintf("%s-%s.tgz", build.Name, build.Version)) + + // If the path of the returned build equals the cache path, + // there are no changes to the chart + if build.Path == cachedChart { + // Ensure hostname is updated + if chart.GetArtifact().URL != newArtifact.URL { r.Storage.SetArtifactURL(chart.GetArtifact()) chart.Status.URL = r.Storage.SetHostname(chart.Status.URL) } @@ -371,362 +386,106 @@ func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context, } defer unlock() - // Attempt to download the chart - res, err := chartRepo.DownloadChart(chartVer) - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err - } - tmpFile, err := os.CreateTemp("", fmt.Sprintf("%s-%s-", chart.Namespace, chart.Name)) - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err - } - defer os.RemoveAll(tmpFile.Name()) - if _, err = io.Copy(tmpFile, res); err != nil { - tmpFile.Close() - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err - } - tmpFile.Close() - - // Check if we need to repackage the chart with the declared defaults files. - var ( - pkgPath = tmpFile.Name() - readyReason = sourcev1.ChartPullSucceededReason - readyMessage = fmt.Sprintf("Fetched revision: %s", newArtifact.Revision) - ) - - switch { - case len(chart.GetValuesFiles()) > 0: - valuesMap := make(map[string]interface{}) - - // Load the chart - helmChart, err := loader.LoadFile(pkgPath) - if err != nil { - err = fmt.Errorf("load chart error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - - for _, v := range chart.GetValuesFiles() { - if v == "values.yaml" { - valuesMap = transform.MergeMaps(valuesMap, helmChart.Values) - continue - } - - var valuesData []byte - cfn := filepath.Clean(v) - for _, f := range helmChart.Files { - if f.Name == cfn { - valuesData = f.Data - break - } - } - if valuesData == nil { - err = fmt.Errorf("invalid values file path: %s", v) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - - yamlMap := make(map[string]interface{}) - err = yaml.Unmarshal(valuesData, &yamlMap) - if err != nil { - err = fmt.Errorf("unmarshaling values from %s failed: %w", v, err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - - valuesMap = transform.MergeMaps(valuesMap, yamlMap) - } - - yamlBytes, err := yaml.Marshal(valuesMap) - if err != nil { - err = fmt.Errorf("marshaling values failed: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err - } - - // Overwrite values file - if changed, err := helm.OverwriteChartDefaultValues(helmChart, yamlBytes); err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err - } else if !changed { - break - } - - // Create temporary working directory - tmpDir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s-", chart.Namespace, chart.Name)) - if err != nil { - err = fmt.Errorf("tmp dir error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - defer os.RemoveAll(tmpDir) - - // Package the chart with the new default values - pkgPath, err = chartutil.Save(helmChart, tmpDir) - if err != nil { - err = fmt.Errorf("chart package error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err - } - - // Copy the packaged chart to the artifact path - if err := r.Storage.CopyFromPath(&newArtifact, pkgPath); err != nil { - err = fmt.Errorf("failed to write chart package to storage: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - - readyMessage = fmt.Sprintf("Fetched and packaged revision: %s", newArtifact.Revision) - readyReason = sourcev1.ChartPackageSucceededReason - } - - // Write artifact to storage - if err := r.Storage.CopyFromPath(&newArtifact, pkgPath); err != nil { - err = fmt.Errorf("unable to write chart file: %w", err) + // Copy the packaged chart to the artifact path + if err = r.Storage.CopyFromPath(&newArtifact, build.Path); err != nil { + err = fmt.Errorf("failed to write chart package to storage: %w", err) return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err } // Update symlink - chartUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", chartVer.Name)) + cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", build.Name)) if err != nil { err = fmt.Errorf("storage error: %w", err) return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err } - - return sourcev1.HelmChartReady(chart, newArtifact, chartUrl, readyReason, readyMessage), nil + return sourcev1.HelmChartReady(chart, newArtifact, cUrl, sourcev1.ChartPullSucceededReason, build.Summary()), nil } -func (r *HelmChartReconciler) reconcileFromTarballArtifact(ctx context.Context, - artifact sourcev1.Artifact, chart sourcev1.HelmChart, force bool) (sourcev1.HelmChart, error) { - // Create temporary working directory - tmpDir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s-", chart.Namespace, chart.Name)) - if err != nil { - err = fmt.Errorf("tmp dir error: %w", err) +func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source sourcev1.Artifact, + chart sourcev1.HelmChart, workDir string, force bool) (sourcev1.HelmChart, error) { + // Create temporary working directory to untar into + sourceDir := filepath.Join(workDir, "source") + if err := os.Mkdir(sourceDir, 0700); err != nil { + err = fmt.Errorf("failed to create temporary directory to untar source into: %w", err) return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err } - defer os.RemoveAll(tmpDir) // Open the tarball artifact file and untar files into working directory - f, err := os.Open(r.Storage.LocalPath(artifact)) + f, err := os.Open(r.Storage.LocalPath(source)) if err != nil { err = fmt.Errorf("artifact open error: %w", err) return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err } - if _, err = untar.Untar(f, tmpDir); err != nil { - f.Close() + if _, err = untar.Untar(f, sourceDir); err != nil { + _ = f.Close() err = fmt.Errorf("artifact untar error: %w", err) return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err } - f.Close() - - // Load the chart - chartPath, err := securejoin.SecureJoin(tmpDir, chart.Spec.Chart) - if err != nil { + if err =f.Close(); err != nil { + err = fmt.Errorf("artifact close error: %w", err) return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err } - chartFileInfo, err := os.Stat(chartPath) + + chartPath, err := securejoin.SecureJoin(sourceDir, chart.Spec.Chart) if err != nil { - err = fmt.Errorf("chart location read error: %w", err) return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err } - helmChart, err := loader.Load(chartPath) - if err != nil { - err = fmt.Errorf("load chart error: %w", err) + + // Setup dependency manager + authDir := filepath.Join(workDir, "creds") + if err = os.Mkdir(authDir, 0700); err != nil { + err = fmt.Errorf("failed to create temporaRy directory for dependency credentials: %w", err) return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err } + dm := helm.NewDependencyManager( + helm.WithRepositoryCallback(r.getNamespacedChartRepositoryCallback(ctx, authDir, chart.GetNamespace())), + ) + defer dm.Clear() - v, err := semver.NewVersion(helmChart.Metadata.Version) - if err != nil { - err = fmt.Errorf("semver parse error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + // Get any cached chart + var cachedChart string + if artifact := chart.Status.Artifact; artifact != nil { + cachedChart = artifact.Path + } + + buildsOpts := helm.BuildOptions{ + ValueFiles: chart.GetValuesFiles(), + CachedChart: cachedChart, + Force: force, } - version := v.String() + // Add revision metadata to chart build if chart.Spec.ReconcileStrategy == sourcev1.ReconcileStrategyRevision { // Isolate the commit SHA from GitRepository type artifacts by removing the branch/ prefix. - splitRev := strings.Split(artifact.Revision, "/") - v, err := v.SetMetadata(splitRev[len(splitRev)-1]) - if err != nil { - err = fmt.Errorf("semver parse error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } + splitRev := strings.Split(source.Revision, "/") + buildsOpts.VersionMetadata = splitRev[len(splitRev)-1] + } - version = v.String() - helmChart.Metadata.Version = v.String() + // Build chart + chartB := helm.NewLocalChartBuilder(dm) + build, err := chartB.Build(ctx, helm.LocalChartReference{BaseDir: sourceDir, Path: chartPath}, filepath.Join(workDir, "chart.tgz"), buildsOpts) + if err != nil { + return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err } - // Return early if the revision is still the same as the current chart artifact - newArtifact := r.Storage.NewArtifactFor(chart.Kind, chart.ObjectMeta.GetObjectMeta(), version, - fmt.Sprintf("%s-%s.tgz", helmChart.Metadata.Name, version)) - if !force && apimeta.IsStatusConditionTrue(chart.Status.Conditions, meta.ReadyCondition) && chart.GetArtifact().HasRevision(newArtifact.Revision) { - if newArtifact.URL != artifact.URL { + newArtifact := r.Storage.NewArtifactFor(chart.Kind, chart.GetObjectMeta(), build.Version, + fmt.Sprintf("%s-%s.tgz", build.Name, build.Version)) + + // If the path of the returned build equals the cache path, + // there are no changes to the chart + if build.Path == cachedChart { + // Ensure hostname is updated + if chart.GetArtifact().URL != newArtifact.URL { r.Storage.SetArtifactURL(chart.GetArtifact()) chart.Status.URL = r.Storage.SetHostname(chart.Status.URL) } return chart, nil } - // Either (re)package the chart with the declared default values file, - // or write the chart directly to storage. - pkgPath := chartPath - isValuesFileOverriden := false - if len(chart.GetValuesFiles()) > 0 { - valuesMap := make(map[string]interface{}) - for _, v := range chart.GetValuesFiles() { - srcPath, err := securejoin.SecureJoin(tmpDir, v) - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - if f, err := os.Stat(srcPath); os.IsNotExist(err) || !f.Mode().IsRegular() { - err = fmt.Errorf("invalid values file path: %s", v) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - - valuesData, err := os.ReadFile(srcPath) - if err != nil { - err = fmt.Errorf("failed to read from values file '%s': %w", v, err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - - yamlMap := make(map[string]interface{}) - err = yaml.Unmarshal(valuesData, &yamlMap) - if err != nil { - err = fmt.Errorf("unmarshaling values from %s failed: %w", v, err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - - valuesMap = transform.MergeMaps(valuesMap, yamlMap) - } - - yamlBytes, err := yaml.Marshal(valuesMap) - if err != nil { - err = fmt.Errorf("marshaling values failed: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err - } - - isValuesFileOverriden, err = helm.OverwriteChartDefaultValues(helmChart, yamlBytes) - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err - } - } - - isDir := chartFileInfo.IsDir() - switch { - case isDir: - // Determine chart dependencies - deps := helmChart.Dependencies() - reqs := helmChart.Metadata.Dependencies - lock := helmChart.Lock - if lock != nil { - // Load from lockfile if exists - reqs = lock.Dependencies - } - var dwr []*helm.DependencyWithRepository - for _, dep := range reqs { - // Exclude existing dependencies - found := false - for _, existing := range deps { - if existing.Name() == dep.Name { - found = true - } - } - if found { - continue - } - - // Continue loop if file scheme detected - if dep.Repository == "" || strings.HasPrefix(dep.Repository, "file://") { - dwr = append(dwr, &helm.DependencyWithRepository{ - Dependency: dep, - Repository: nil, - }) - continue - } - - // Discover existing HelmRepository by URL - repository, err := r.resolveDependencyRepository(ctx, dep, chart.Namespace) - if err != nil { - repository = &sourcev1.HelmRepository{ - Spec: sourcev1.HelmRepositorySpec{ - URL: dep.Repository, - Timeout: &metav1.Duration{Duration: 60 * time.Second}, - }, - } - } - - // Configure ChartRepository getter options - clientOpts := []getter.Option{ - getter.WithURL(repository.Spec.URL), - getter.WithTimeout(repository.Spec.Timeout.Duration), - getter.WithPassCredentialsAll(repository.Spec.PassCredentials), - } - if secret, err := r.getHelmRepositorySecret(ctx, repository); err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.AuthenticationFailedReason, err.Error()), err - } else if secret != nil { - opts, cleanup, err := helm.ClientOptionsFromSecret(*secret) - if err != nil { - err = fmt.Errorf("auth options error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.AuthenticationFailedReason, err.Error()), err - } - defer cleanup() - clientOpts = append(clientOpts, opts...) - } - - // Initialize the chart repository and load the index file - chartRepo, err := helm.NewChartRepository(repository.Spec.URL, r.Getters, clientOpts) - if err != nil { - switch err.(type) { - case *url.Error: - return sourcev1.HelmChartNotReady(chart, sourcev1.URLInvalidReason, err.Error()), err - default: - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err - } - } - if repository.Status.Artifact != nil { - indexFile, err := os.Open(r.Storage.LocalPath(*repository.GetArtifact())) - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - b, err := io.ReadAll(indexFile) - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err - } - if err = chartRepo.LoadIndex(b); err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err - } - } else { - // Download index - err = chartRepo.DownloadIndex() - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err - } - } - - dwr = append(dwr, &helm.DependencyWithRepository{ - Dependency: dep, - Repository: chartRepo, - }) - } - - // Construct dependencies for chart if any - if len(dwr) > 0 { - dm := &helm.DependencyManager{ - WorkingDir: tmpDir, - ChartPath: chart.Spec.Chart, - Chart: helmChart, - Dependencies: dwr, - } - err = dm.Build(ctx) - if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err - } - } - - fallthrough - case isValuesFileOverriden: - pkgPath, err = chartutil.Save(helmChart, tmpDir) - if err != nil { - err = fmt.Errorf("chart package error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err - } - } - // Ensure artifact directory exists err = r.Storage.MkdirAll(newArtifact) if err != nil { - err = fmt.Errorf("unable to create artifact directory: %w", err) + err = fmt.Errorf("unable to create chart directory: %w", err) return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err } @@ -739,20 +498,59 @@ func (r *HelmChartReconciler) reconcileFromTarballArtifact(ctx context.Context, defer unlock() // Copy the packaged chart to the artifact path - if err := r.Storage.CopyFromPath(&newArtifact, pkgPath); err != nil { + if err = r.Storage.CopyFromPath(&newArtifact, build.Path); err != nil { err = fmt.Errorf("failed to write chart package to storage: %w", err) return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err } // Update symlink - cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", helmChart.Metadata.Name)) + cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", chart.Name)) if err != nil { err = fmt.Errorf("storage error: %w", err) return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err } - message := fmt.Sprintf("Fetched and packaged revision: %s", newArtifact.Revision) - return sourcev1.HelmChartReady(chart, newArtifact, cUrl, sourcev1.ChartPackageSucceededReason, message), nil + return sourcev1.HelmChartReady(chart, newArtifact, cUrl, sourcev1.ChartPackageSucceededReason, build.Summary()), nil +} + +// TODO(hidde): factor out to helper? +func (r *HelmChartReconciler) getNamespacedChartRepositoryCallback(ctx context.Context, dir, namespace string) helm.GetChartRepositoryCallback { + return func(url string) (*helm.ChartRepository, error) { + repo, err := r.resolveDependencyRepository(ctx, url, namespace) + if err != nil { + if errors.ReasonForError(err) != metav1.StatusReasonUnknown { + return nil, err + } + repo = &sourcev1.HelmRepository{ + Spec: sourcev1.HelmRepositorySpec{ + URL: url, + Timeout: &metav1.Duration{Duration: 60 * time.Second}, + }, + } + } + clientOpts := []getter.Option{ + getter.WithURL(repo.Spec.URL), + getter.WithTimeout(repo.Spec.Timeout.Duration), + getter.WithPassCredentialsAll(repo.Spec.PassCredentials), + } + if secret, err := r.getHelmRepositorySecret(ctx, repo); err != nil { + return nil, err + } else if secret != nil { + opts, err := helm.ClientOptionsFromSecret(dir, *secret) + if err != nil { + return nil, err + } + clientOpts = append(clientOpts, opts...) + } + chartRepo, err := helm.NewChartRepository(repo.Spec.URL, "", r.Getters, clientOpts) + if err != nil { + return nil, err + } + if repo.Status.Artifact != nil { + chartRepo.CachePath = r.Storage.LocalPath(*repo.GetArtifact()) + } + return chartRepo, nil + } } func (r *HelmChartReconciler) reconcileDelete(ctx context.Context, chart sourcev1.HelmChart) (ctrl.Result, error) { @@ -880,15 +678,10 @@ func (r *HelmChartReconciler) indexHelmChartBySource(o client.Object) []string { return []string{fmt.Sprintf("%s/%s", hc.Spec.SourceRef.Kind, hc.Spec.SourceRef.Name)} } -func (r *HelmChartReconciler) resolveDependencyRepository(ctx context.Context, dep *helmchart.Dependency, namespace string) (*sourcev1.HelmRepository, error) { - u := helm.NormalizeChartRepositoryURL(dep.Repository) - if u == "" { - return nil, fmt.Errorf("invalid repository URL") - } - +func (r *HelmChartReconciler) resolveDependencyRepository(ctx context.Context, url string, namespace string) (*sourcev1.HelmRepository, error) { listOpts := []client.ListOption{ client.InNamespace(namespace), - client.MatchingFields{sourcev1.HelmRepositoryURLIndexKey: u}, + client.MatchingFields{sourcev1.HelmRepositoryURLIndexKey: url}, } var list sourcev1.HelmRepositoryList err := r.Client.List(ctx, &list, listOpts...) @@ -898,8 +691,7 @@ func (r *HelmChartReconciler) resolveDependencyRepository(ctx context.Context, d if len(list.Items) > 0 { return &list.Items[0], nil } - - return nil, fmt.Errorf("no HelmRepository found") + return nil, fmt.Errorf("no HelmRepository found for '%s' in '%s' namespace", url, namespace) } func (r *HelmChartReconciler) getHelmRepositorySecret(ctx context.Context, repository *sourcev1.HelmRepository) (*corev1.Secret, error) { @@ -917,7 +709,6 @@ func (r *HelmChartReconciler) getHelmRepositorySecret(ctx context.Context, repos } return &secret, nil } - return nil, nil } diff --git a/controllers/helmchart_controller_test.go b/controllers/helmchart_controller_test.go index 35462d467..ceb30842f 100644 --- a/controllers/helmchart_controller_test.go +++ b/controllers/helmchart_controller_test.go @@ -732,6 +732,7 @@ var _ = Describe("HelmChartReconciler", func() { }, timeout, interval).Should(BeTrue()) helmChart, err := loader.Load(storage.LocalPath(*now.Status.Artifact)) Expect(err).NotTo(HaveOccurred()) + Expect(helmChart.Values).ToNot(BeNil()) Expect(helmChart.Values["testDefault"]).To(BeTrue()) Expect(helmChart.Values["testOverride"]).To(BeFalse()) diff --git a/controllers/helmrepository_controller.go b/controllers/helmrepository_controller.go index d7fb57e58..794a912e3 100644 --- a/controllers/helmrepository_controller.go +++ b/controllers/helmrepository_controller.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "net/url" + "os" "time" "github.com/fluxcd/pkg/apis/meta" @@ -186,12 +187,18 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sou return sourcev1.HelmRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err } - opts, cleanup, err := helm.ClientOptionsFromSecret(secret) + authDir, err := os.MkdirTemp("", "helm-repository-") + if err != nil { + err = fmt.Errorf("failed to create temporary working directory for credentials: %w", err) + return sourcev1.HelmRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err + } + defer os.RemoveAll(authDir) + + opts, err := helm.ClientOptionsFromSecret(authDir, secret) if err != nil { err = fmt.Errorf("auth options error: %w", err) return sourcev1.HelmRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err } - defer cleanup() clientOpts = append(clientOpts, opts...) } From 7d0f79f41b84efea5d6b0fd6cab64ea58daf149d Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Mon, 15 Nov 2021 22:31:33 +0100 Subject: [PATCH 08/23] internal/helm: divide into subpackages With all the logic that used to reside in the `controllers` package factored into this package, it became cluttered. This commit tries to bring a bit more structure in place. Signed-off-by: Hidde Beydals --- controllers/helmchart_controller.go | 146 +++++++++--------- controllers/helmrepository_controller.go | 78 +++++----- .../{chart_builder.go => chart/builder.go} | 66 ++++---- .../builder_local.go} | 21 +-- .../builder_local_test.go} | 8 +- .../builder_remote.go} | 25 +-- .../builder_remote_test.go} | 8 +- .../builder_test.go} | 10 +- .../helm/{ => chart}/dependency_manager.go | 61 ++++---- .../{ => chart}/dependency_manager_test.go | 85 ++++------ internal/helm/{chart.go => chart/metadata.go} | 12 +- .../{chart_test.go => chart/metadata_test.go} | 27 +++- internal/helm/{ => getter}/getter.go | 2 +- internal/helm/{ => getter}/getter_test.go | 2 +- internal/helm/getter/mock.go | 41 +++++ .../chart_repository.go} | 10 +- .../chart_repository_test.go} | 46 +++--- internal/helm/{ => repository}/utils.go | 9 +- internal/helm/repository/utils_test.go | 44 ++++++ internal/helm/utils_test.go | 60 ------- 20 files changed, 397 insertions(+), 364 deletions(-) rename internal/helm/{chart_builder.go => chart/builder.go} (70%) rename internal/helm/{chart_builder_local.go => chart/builder_local.go} (90%) rename internal/helm/{chart_builder_local_test.go => chart/builder_local_test.go} (96%) rename internal/helm/{chart_builder_remote.go => chart/builder_remote.go} (91%) rename internal/helm/{chart_builder_remote_test.go => chart/builder_remote_test.go} (92%) rename internal/helm/{chart_builder_test.go => chart/builder_test.go} (89%) rename internal/helm/{ => chart}/dependency_manager.go (81%) rename internal/helm/{ => chart}/dependency_manager_test.go (84%) rename internal/helm/{chart.go => chart/metadata.go} (96%) rename internal/helm/{chart_test.go => chart/metadata_test.go} (85%) rename internal/helm/{ => getter}/getter.go (99%) rename internal/helm/{ => getter}/getter_test.go (99%) create mode 100644 internal/helm/getter/mock.go rename internal/helm/{repository.go => repository/chart_repository.go} (98%) rename internal/helm/{repository_test.go => repository/chart_repository_test.go} (93%) rename internal/helm/{ => repository}/utils.go (77%) create mode 100644 internal/helm/repository/utils_test.go delete mode 100644 internal/helm/utils_test.go diff --git a/controllers/helmchart_controller.go b/controllers/helmchart_controller.go index bcb8f8e79..d31f6c2bb 100644 --- a/controllers/helmchart_controller.go +++ b/controllers/helmchart_controller.go @@ -28,7 +28,7 @@ import ( securejoin "github.com/cyphar/filepath-securejoin" "github.com/go-logr/logr" - "helm.sh/helm/v3/pkg/getter" + extgetter "helm.sh/helm/v3/pkg/getter" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" @@ -54,7 +54,9 @@ import ( "github.com/fluxcd/pkg/untar" sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" - "github.com/fluxcd/source-controller/internal/helm" + "github.com/fluxcd/source-controller/internal/helm/chart" + "github.com/fluxcd/source-controller/internal/helm/getter" + "github.com/fluxcd/source-controller/internal/helm/repository" ) // +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=helmcharts,verbs=get;list;watch;create;update;patch;delete @@ -67,7 +69,7 @@ type HelmChartReconciler struct { client.Client Scheme *runtime.Scheme Storage *Storage - Getters getter.Providers + Getters extgetter.Providers EventRecorder kuberecorder.EventRecorder ExternalEventRecorder *events.Recorder MetricsRecorder *metrics.Recorder @@ -304,218 +306,218 @@ func (r *HelmChartReconciler) getSource(ctx context.Context, chart sourcev1.Helm return source, nil } -func (r *HelmChartReconciler) fromHelmRepository(ctx context.Context, repository sourcev1.HelmRepository, - chart sourcev1.HelmChart, workDir string, force bool) (sourcev1.HelmChart, error) { - // Configure ChartRepository getter options - clientOpts := []getter.Option{ - getter.WithURL(repository.Spec.URL), - getter.WithTimeout(repository.Spec.Timeout.Duration), - getter.WithPassCredentialsAll(repository.Spec.PassCredentials), +func (r *HelmChartReconciler) fromHelmRepository(ctx context.Context, repo sourcev1.HelmRepository, c sourcev1.HelmChart, + workDir string, force bool) (sourcev1.HelmChart, error) { + // Configure Index getter options + clientOpts := []extgetter.Option{ + extgetter.WithURL(repo.Spec.URL), + extgetter.WithTimeout(repo.Spec.Timeout.Duration), + extgetter.WithPassCredentialsAll(repo.Spec.PassCredentials), } - if secret, err := r.getHelmRepositorySecret(ctx, &repository); err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.AuthenticationFailedReason, err.Error()), err + if secret, err := r.getHelmRepositorySecret(ctx, &repo); err != nil { + return sourcev1.HelmChartNotReady(c, sourcev1.AuthenticationFailedReason, err.Error()), err } else if secret != nil { // Create temporary working directory for credentials authDir := filepath.Join(workDir, "creds") if err := os.Mkdir(authDir, 0700); err != nil { err = fmt.Errorf("failed to create temporary directory for repository credentials: %w", err) } - opts, err := helm.ClientOptionsFromSecret(authDir, *secret) + opts, err := getter.ClientOptionsFromSecret(authDir, *secret) if err != nil { - err = fmt.Errorf("failed to create client options for HelmRepository '%s': %w", repository.Name, err) - return sourcev1.HelmChartNotReady(chart, sourcev1.AuthenticationFailedReason, err.Error()), err + err = fmt.Errorf("failed to create client options for HelmRepository '%s': %w", repo.Name, err) + return sourcev1.HelmChartNotReady(c, sourcev1.AuthenticationFailedReason, err.Error()), err } clientOpts = append(clientOpts, opts...) } // Initialize the chart repository - chartRepo, err := helm.NewChartRepository(repository.Spec.URL, r.Storage.LocalPath(*repository.GetArtifact()), r.Getters, clientOpts) + chartRepo, err := repository.NewChartRepository(repo.Spec.URL, r.Storage.LocalPath(*repo.GetArtifact()), r.Getters, clientOpts) if err != nil { switch err.(type) { case *url.Error: - return sourcev1.HelmChartNotReady(chart, sourcev1.URLInvalidReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.URLInvalidReason, err.Error()), err default: - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.ChartPullFailedReason, err.Error()), err } } var cachedChart string - if artifact := chart.GetArtifact(); artifact != nil { + if artifact := c.GetArtifact(); artifact != nil { cachedChart = artifact.Path } // Build the chart - cBuilder := helm.NewRemoteChartBuilder(chartRepo) - ref := helm.RemoteChartReference{Name: chart.Spec.Chart, Version: chart.Spec.Version} - opts := helm.BuildOptions{ - ValueFiles: chart.GetValuesFiles(), + cBuilder := chart.NewRemoteBuilder(chartRepo) + ref := chart.RemoteReference{Name: c.Spec.Chart, Version: c.Spec.Version} + opts := chart.BuildOptions{ + ValueFiles: c.GetValuesFiles(), CachedChart: cachedChart, Force: force, } build, err := cBuilder.Build(ctx, ref, filepath.Join(workDir, "chart.tgz"), opts) if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.ChartPullFailedReason, err.Error()), err } - newArtifact := r.Storage.NewArtifactFor(chart.Kind, chart.GetObjectMeta(), build.Version, + newArtifact := r.Storage.NewArtifactFor(c.Kind, c.GetObjectMeta(), build.Version, fmt.Sprintf("%s-%s.tgz", build.Name, build.Version)) // If the path of the returned build equals the cache path, // there are no changes to the chart if build.Path == cachedChart { // Ensure hostname is updated - if chart.GetArtifact().URL != newArtifact.URL { - r.Storage.SetArtifactURL(chart.GetArtifact()) - chart.Status.URL = r.Storage.SetHostname(chart.Status.URL) + if c.GetArtifact().URL != newArtifact.URL { + r.Storage.SetArtifactURL(c.GetArtifact()) + c.Status.URL = r.Storage.SetHostname(c.Status.URL) } - return chart, nil + return c, nil } // Ensure artifact directory exists err = r.Storage.MkdirAll(newArtifact) if err != nil { err = fmt.Errorf("unable to create chart directory: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } // Acquire a lock for the artifact unlock, err := r.Storage.Lock(newArtifact) if err != nil { err = fmt.Errorf("unable to acquire lock: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } defer unlock() // Copy the packaged chart to the artifact path if err = r.Storage.CopyFromPath(&newArtifact, build.Path); err != nil { err = fmt.Errorf("failed to write chart package to storage: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } // Update symlink cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", build.Name)) if err != nil { err = fmt.Errorf("storage error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } - return sourcev1.HelmChartReady(chart, newArtifact, cUrl, sourcev1.ChartPullSucceededReason, build.Summary()), nil + return sourcev1.HelmChartReady(c, newArtifact, cUrl, sourcev1.ChartPullSucceededReason, build.Summary()), nil } -func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source sourcev1.Artifact, - chart sourcev1.HelmChart, workDir string, force bool) (sourcev1.HelmChart, error) { +func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source sourcev1.Artifact, c sourcev1.HelmChart, + workDir string, force bool) (sourcev1.HelmChart, error) { // Create temporary working directory to untar into sourceDir := filepath.Join(workDir, "source") if err := os.Mkdir(sourceDir, 0700); err != nil { err = fmt.Errorf("failed to create temporary directory to untar source into: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } // Open the tarball artifact file and untar files into working directory f, err := os.Open(r.Storage.LocalPath(source)) if err != nil { err = fmt.Errorf("artifact open error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } if _, err = untar.Untar(f, sourceDir); err != nil { _ = f.Close() err = fmt.Errorf("artifact untar error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } if err =f.Close(); err != nil { err = fmt.Errorf("artifact close error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } - chartPath, err := securejoin.SecureJoin(sourceDir, chart.Spec.Chart) + chartPath, err := securejoin.SecureJoin(sourceDir, c.Spec.Chart) if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } // Setup dependency manager authDir := filepath.Join(workDir, "creds") if err = os.Mkdir(authDir, 0700); err != nil { err = fmt.Errorf("failed to create temporaRy directory for dependency credentials: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } - dm := helm.NewDependencyManager( - helm.WithRepositoryCallback(r.getNamespacedChartRepositoryCallback(ctx, authDir, chart.GetNamespace())), + dm := chart.NewDependencyManager( + chart.WithRepositoryCallback(r.getNamespacedChartRepositoryCallback(ctx, authDir, c.GetNamespace())), ) defer dm.Clear() // Get any cached chart var cachedChart string - if artifact := chart.Status.Artifact; artifact != nil { + if artifact := c.Status.Artifact; artifact != nil { cachedChart = artifact.Path } - buildsOpts := helm.BuildOptions{ - ValueFiles: chart.GetValuesFiles(), + buildsOpts := chart.BuildOptions{ + ValueFiles: c.GetValuesFiles(), CachedChart: cachedChart, Force: force, } // Add revision metadata to chart build - if chart.Spec.ReconcileStrategy == sourcev1.ReconcileStrategyRevision { + if c.Spec.ReconcileStrategy == sourcev1.ReconcileStrategyRevision { // Isolate the commit SHA from GitRepository type artifacts by removing the branch/ prefix. splitRev := strings.Split(source.Revision, "/") buildsOpts.VersionMetadata = splitRev[len(splitRev)-1] } // Build chart - chartB := helm.NewLocalChartBuilder(dm) - build, err := chartB.Build(ctx, helm.LocalChartReference{BaseDir: sourceDir, Path: chartPath}, filepath.Join(workDir, "chart.tgz"), buildsOpts) + chartB := chart.NewLocalBuilder(dm) + build, err := chartB.Build(ctx, chart.LocalReference{BaseDir: sourceDir, Path: chartPath}, filepath.Join(workDir, "chart.tgz"), buildsOpts) if err != nil { - return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPackageFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.ChartPackageFailedReason, err.Error()), err } - newArtifact := r.Storage.NewArtifactFor(chart.Kind, chart.GetObjectMeta(), build.Version, + newArtifact := r.Storage.NewArtifactFor(c.Kind, c.GetObjectMeta(), build.Version, fmt.Sprintf("%s-%s.tgz", build.Name, build.Version)) // If the path of the returned build equals the cache path, // there are no changes to the chart if build.Path == cachedChart { // Ensure hostname is updated - if chart.GetArtifact().URL != newArtifact.URL { - r.Storage.SetArtifactURL(chart.GetArtifact()) - chart.Status.URL = r.Storage.SetHostname(chart.Status.URL) + if c.GetArtifact().URL != newArtifact.URL { + r.Storage.SetArtifactURL(c.GetArtifact()) + c.Status.URL = r.Storage.SetHostname(c.Status.URL) } - return chart, nil + return c, nil } // Ensure artifact directory exists err = r.Storage.MkdirAll(newArtifact) if err != nil { err = fmt.Errorf("unable to create chart directory: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } // Acquire a lock for the artifact unlock, err := r.Storage.Lock(newArtifact) if err != nil { err = fmt.Errorf("unable to acquire lock: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } defer unlock() // Copy the packaged chart to the artifact path if err = r.Storage.CopyFromPath(&newArtifact, build.Path); err != nil { err = fmt.Errorf("failed to write chart package to storage: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } // Update symlink - cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", chart.Name)) + cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", build.Name)) if err != nil { err = fmt.Errorf("storage error: %w", err) - return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } - return sourcev1.HelmChartReady(chart, newArtifact, cUrl, sourcev1.ChartPackageSucceededReason, build.Summary()), nil + return sourcev1.HelmChartReady(c, newArtifact, cUrl, sourcev1.ChartPackageSucceededReason, build.Summary()), nil } // TODO(hidde): factor out to helper? -func (r *HelmChartReconciler) getNamespacedChartRepositoryCallback(ctx context.Context, dir, namespace string) helm.GetChartRepositoryCallback { - return func(url string) (*helm.ChartRepository, error) { +func (r *HelmChartReconciler) getNamespacedChartRepositoryCallback(ctx context.Context, dir, namespace string) chart.GetChartRepositoryCallback { + return func(url string) (*repository.ChartRepository, error) { repo, err := r.resolveDependencyRepository(ctx, url, namespace) if err != nil { if errors.ReasonForError(err) != metav1.StatusReasonUnknown { @@ -528,21 +530,21 @@ func (r *HelmChartReconciler) getNamespacedChartRepositoryCallback(ctx context.C }, } } - clientOpts := []getter.Option{ - getter.WithURL(repo.Spec.URL), - getter.WithTimeout(repo.Spec.Timeout.Duration), - getter.WithPassCredentialsAll(repo.Spec.PassCredentials), + clientOpts := []extgetter.Option{ + extgetter.WithURL(repo.Spec.URL), + extgetter.WithTimeout(repo.Spec.Timeout.Duration), + extgetter.WithPassCredentialsAll(repo.Spec.PassCredentials), } if secret, err := r.getHelmRepositorySecret(ctx, repo); err != nil { return nil, err } else if secret != nil { - opts, err := helm.ClientOptionsFromSecret(dir, *secret) + opts, err := getter.ClientOptionsFromSecret(dir, *secret) if err != nil { return nil, err } clientOpts = append(clientOpts, opts...) } - chartRepo, err := helm.NewChartRepository(repo.Spec.URL, "", r.Getters, clientOpts) + chartRepo, err := repository.NewChartRepository(repo.Spec.URL, "", r.Getters, clientOpts) if err != nil { return nil, err } @@ -663,7 +665,7 @@ func (r *HelmChartReconciler) indexHelmRepositoryByURL(o client.Object) []string if !ok { panic(fmt.Sprintf("Expected a HelmRepository, got %T", o)) } - u := helm.NormalizeChartRepositoryURL(repo.Spec.URL) + u := repository.NormalizeURL(repo.Spec.URL) if u != "" { return []string{u} } diff --git a/controllers/helmrepository_controller.go b/controllers/helmrepository_controller.go index 794a912e3..8ab87201d 100644 --- a/controllers/helmrepository_controller.go +++ b/controllers/helmrepository_controller.go @@ -23,12 +23,8 @@ import ( "os" "time" - "github.com/fluxcd/pkg/apis/meta" - "github.com/fluxcd/pkg/runtime/events" - "github.com/fluxcd/pkg/runtime/metrics" - "github.com/fluxcd/pkg/runtime/predicates" "github.com/go-logr/logr" - "helm.sh/helm/v3/pkg/getter" + extgetter "helm.sh/helm/v3/pkg/getter" corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -42,8 +38,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/predicate" + "github.com/fluxcd/pkg/apis/meta" + "github.com/fluxcd/pkg/runtime/events" + "github.com/fluxcd/pkg/runtime/metrics" + "github.com/fluxcd/pkg/runtime/predicates" + + "github.com/fluxcd/source-controller/internal/helm/getter" + "github.com/fluxcd/source-controller/internal/helm/repository" sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" - "github.com/fluxcd/source-controller/internal/helm" ) // +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=helmrepositories,verbs=get;list;watch;create;update;patch;delete @@ -56,7 +58,7 @@ type HelmRepositoryReconciler struct { client.Client Scheme *runtime.Scheme Storage *Storage - Getters getter.Providers + Getters extgetter.Providers EventRecorder kuberecorder.EventRecorder ExternalEventRecorder *events.Recorder MetricsRecorder *metrics.Recorder @@ -168,74 +170,74 @@ func (r *HelmRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reque return ctrl.Result{RequeueAfter: repository.GetInterval().Duration}, nil } -func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sourcev1.HelmRepository) (sourcev1.HelmRepository, error) { - clientOpts := []getter.Option{ - getter.WithURL(repository.Spec.URL), - getter.WithTimeout(repository.Spec.Timeout.Duration), - getter.WithPassCredentialsAll(repository.Spec.PassCredentials), +func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repo sourcev1.HelmRepository) (sourcev1.HelmRepository, error) { + clientOpts := []extgetter.Option{ + extgetter.WithURL(repo.Spec.URL), + extgetter.WithTimeout(repo.Spec.Timeout.Duration), + extgetter.WithPassCredentialsAll(repo.Spec.PassCredentials), } - if repository.Spec.SecretRef != nil { + if repo.Spec.SecretRef != nil { name := types.NamespacedName{ - Namespace: repository.GetNamespace(), - Name: repository.Spec.SecretRef.Name, + Namespace: repo.GetNamespace(), + Name: repo.Spec.SecretRef.Name, } var secret corev1.Secret err := r.Client.Get(ctx, name, &secret) if err != nil { err = fmt.Errorf("auth secret error: %w", err) - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.AuthenticationFailedReason, err.Error()), err } authDir, err := os.MkdirTemp("", "helm-repository-") if err != nil { err = fmt.Errorf("failed to create temporary working directory for credentials: %w", err) - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.AuthenticationFailedReason, err.Error()), err } defer os.RemoveAll(authDir) - opts, err := helm.ClientOptionsFromSecret(authDir, secret) + opts, err := getter.ClientOptionsFromSecret(authDir, secret) if err != nil { err = fmt.Errorf("auth options error: %w", err) - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.AuthenticationFailedReason, err.Error()), err } clientOpts = append(clientOpts, opts...) } - chartRepo, err := helm.NewChartRepository(repository.Spec.URL, "", r.Getters, clientOpts) + chartRepo, err := repository.NewChartRepository(repo.Spec.URL, "", r.Getters, clientOpts) if err != nil { switch err.(type) { case *url.Error: - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.URLInvalidReason, err.Error()), err + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.URLInvalidReason, err.Error()), err default: - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.IndexationFailedReason, err.Error()), err + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.IndexationFailedReason, err.Error()), err } } revision, err := chartRepo.CacheIndex() if err != nil { err = fmt.Errorf("failed to download repository index: %w", err) - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.IndexationFailedReason, err.Error()), err + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.IndexationFailedReason, err.Error()), err } defer chartRepo.RemoveCache() - artifact := r.Storage.NewArtifactFor(repository.Kind, - repository.ObjectMeta.GetObjectMeta(), + artifact := r.Storage.NewArtifactFor(repo.Kind, + repo.ObjectMeta.GetObjectMeta(), revision, fmt.Sprintf("index-%s.yaml", revision)) // Return early on unchanged index - if apimeta.IsStatusConditionTrue(repository.Status.Conditions, meta.ReadyCondition) && - repository.GetArtifact().HasRevision(artifact.Revision) { - if artifact.URL != repository.GetArtifact().URL { - r.Storage.SetArtifactURL(repository.GetArtifact()) - repository.Status.URL = r.Storage.SetHostname(repository.Status.URL) + if apimeta.IsStatusConditionTrue(repo.Status.Conditions, meta.ReadyCondition) && + repo.GetArtifact().HasRevision(artifact.Revision) { + if artifact.URL != repo.GetArtifact().URL { + r.Storage.SetArtifactURL(repo.GetArtifact()) + repo.Status.URL = r.Storage.SetHostname(repo.Status.URL) } - return repository, nil + return repo, nil } // Load the cached repository index to ensure it passes validation if err := chartRepo.LoadFromCache(); err != nil { - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.IndexationFailedReason, err.Error()), err + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.IndexationFailedReason, err.Error()), err } defer chartRepo.Unload() @@ -243,14 +245,14 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sou err = r.Storage.MkdirAll(artifact) if err != nil { err = fmt.Errorf("unable to create repository index directory: %w", err) - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.StorageOperationFailedReason, err.Error()), err } // Acquire lock unlock, err := r.Storage.Lock(artifact) if err != nil { err = fmt.Errorf("unable to acquire lock: %w", err) - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.StorageOperationFailedReason, err.Error()), err } defer unlock() @@ -258,10 +260,10 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sou storageTarget := r.Storage.LocalPath(artifact) if storageTarget == "" { err := fmt.Errorf("failed to calcalute local storage path to store artifact to") - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.StorageOperationFailedReason, err.Error()), err } if err = chartRepo.Index.WriteFile(storageTarget, 0644); err != nil { - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.StorageOperationFailedReason, err.Error()), err } // TODO(hidde): it would be better to make the Storage deal with this artifact.Checksum = chartRepo.Checksum @@ -271,11 +273,11 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sou indexURL, err := r.Storage.Symlink(artifact, "index.yaml") if err != nil { err = fmt.Errorf("storage error: %w", err) - return sourcev1.HelmRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err + return sourcev1.HelmRepositoryNotReady(repo, sourcev1.StorageOperationFailedReason, err.Error()), err } message := fmt.Sprintf("Fetched revision: %s", artifact.Revision) - return sourcev1.HelmRepositoryReady(repository, artifact, indexURL, sourcev1.IndexationSucceededReason, message), nil + return sourcev1.HelmRepositoryReady(repo, artifact, indexURL, sourcev1.IndexationSucceededReason, message), nil } func (r *HelmRepositoryReconciler) reconcileDelete(ctx context.Context, repository sourcev1.HelmRepository) (ctrl.Result, error) { diff --git a/internal/helm/chart_builder.go b/internal/helm/chart/builder.go similarity index 70% rename from internal/helm/chart_builder.go rename to internal/helm/chart/builder.go index 4177983c6..3698d02c1 100644 --- a/internal/helm/chart_builder.go +++ b/internal/helm/chart/builder.go @@ -14,49 +14,51 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helm +package chart import ( "context" "fmt" "os" "path/filepath" + "regexp" "strings" - "github.com/fluxcd/source-controller/internal/fs" helmchart "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chartutil" + + "github.com/fluxcd/source-controller/internal/fs" ) -// ChartReference holds information to locate a chart. -type ChartReference interface { - // Validate returns an error if the ChartReference is not valid according +// Reference holds information to locate a chart. +type Reference interface { + // Validate returns an error if the Reference is not valid according // to the spec of the interface implementation. Validate() error } -// LocalChartReference contains sufficient information to locate a chart on the +// LocalReference contains sufficient information to locate a chart on the // local filesystem. -type LocalChartReference struct { - // BaseDir used as chroot during build operations. +type LocalReference struct { + // WorkDir used as chroot during build operations. // File references are not allowed to traverse outside it. - BaseDir string + WorkDir string // Path of the chart on the local filesystem. Path string } -// Validate returns an error if the LocalChartReference does not have +// Validate returns an error if the LocalReference does not have // a Path set. -func (r LocalChartReference) Validate() error { +func (r LocalReference) Validate() error { if r.Path == "" { return fmt.Errorf("no path set for local chart reference") } return nil } -// RemoteChartReference contains sufficient information to look up a chart in +// RemoteReference contains sufficient information to look up a chart in // a ChartRepository. -type RemoteChartReference struct { +type RemoteReference struct { // Name of the chart. Name string // Version of the chart. @@ -64,25 +66,29 @@ type RemoteChartReference struct { Version string } -// Validate returns an error if the RemoteChartReference does not have +// Validate returns an error if the RemoteReference does not have // a Name set. -func (r RemoteChartReference) Validate() error { +func (r RemoteReference) Validate() error { if r.Name == "" { return fmt.Errorf("no name set for remote chart reference") } + name := regexp.MustCompile("^([-a-z0-9]*)$") + if !name.MatchString(r.Name) { + return fmt.Errorf("invalid chart name '%s': a valid name must be lower case letters and numbers and MAY be separated with dashes (-)", r.Name) + } return nil } -// ChartBuilder is capable of building a (specific) ChartReference. -type ChartBuilder interface { - // Build builds and packages a Helm chart with the given ChartReference - // and BuildOptions and writes it to p. It returns the ChartBuild result, - // or an error. It may return an error for unsupported ChartReference +// Builder is capable of building a (specific) chart Reference. +type Builder interface { + // Build builds and packages a Helm chart with the given Reference + // and BuildOptions and writes it to p. It returns the Build result, + // or an error. It may return an error for unsupported Reference // implementations. - Build(ctx context.Context, ref ChartReference, p string, opts BuildOptions) (*ChartBuild, error) + Build(ctx context.Context, ref Reference, p string, opts BuildOptions) (*Build, error) } -// BuildOptions provides a list of options for ChartBuilder.Build. +// BuildOptions provides a list of options for Builder.Build. type BuildOptions struct { // VersionMetadata can be set to SemVer build metadata as defined in // the spec, and is included during packaging. @@ -109,9 +115,9 @@ func (o BuildOptions) GetValueFiles() []string { return o.ValueFiles } -// ChartBuild contains the ChartBuilder.Build result, including specific +// Build contains the Builder.Build result, including specific // information about the built chart like ResolvedDependencies. -type ChartBuild struct { +type Build struct { // Path is the absolute path to the packaged chart. Path string // Name of the packaged chart. @@ -124,14 +130,14 @@ type ChartBuild struct { // ResolvedDependencies is the number of local and remote dependencies // collected by the DependencyManager before building the chart. ResolvedDependencies int - // Packaged indicates if the ChartBuilder has packaged the chart. + // Packaged indicates if the Builder has packaged the chart. // This can for example be false if ValueFiles is empty and the chart // source was already packaged. Packaged bool } -// Summary returns a human-readable summary of the ChartBuild. -func (b *ChartBuild) Summary() string { +// Summary returns a human-readable summary of the Build. +func (b *Build) Summary() string { if b == nil { return "no chart build" } @@ -155,15 +161,15 @@ func (b *ChartBuild) Summary() string { return s.String() } -// String returns the Path of the ChartBuild. -func (b *ChartBuild) String() string { +// String returns the Path of the Build. +func (b *Build) String() string { if b != nil { return b.Path } return "" } -// packageToPath attempts to package the given chart.Chart to the out filepath. +// packageToPath attempts to package the given chart to the out filepath. func packageToPath(chart *helmchart.Chart, out string) error { o, err := os.MkdirTemp("", "chart-build-*") if err != nil { diff --git a/internal/helm/chart_builder_local.go b/internal/helm/chart/builder_local.go similarity index 90% rename from internal/helm/chart_builder_local.go rename to internal/helm/chart/builder_local.go index 13e5dbe9c..037a2fe18 100644 --- a/internal/helm/chart_builder_local.go +++ b/internal/helm/chart/builder_local.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helm +package chart import ( "context" @@ -24,27 +24,28 @@ import ( "github.com/Masterminds/semver/v3" securejoin "github.com/cyphar/filepath-securejoin" - "github.com/fluxcd/pkg/runtime/transform" "helm.sh/helm/v3/pkg/chart/loader" "sigs.k8s.io/yaml" + + "github.com/fluxcd/pkg/runtime/transform" ) type localChartBuilder struct { dm *DependencyManager } -// NewLocalChartBuilder returns a ChartBuilder capable of building a Helm -// chart with a LocalChartReference. For chart references pointing to a +// NewLocalBuilder returns a Builder capable of building a Helm +// chart with a LocalReference. For chart references pointing to a // directory, the DependencyManager is used to resolve missing local and // remote dependencies. -func NewLocalChartBuilder(dm *DependencyManager) ChartBuilder { +func NewLocalBuilder(dm *DependencyManager) Builder { return &localChartBuilder{ dm: dm, } } -func (b *localChartBuilder) Build(ctx context.Context, ref ChartReference, p string, opts BuildOptions) (*ChartBuild, error) { - localRef, ok := ref.(LocalChartReference) +func (b *localChartBuilder) Build(ctx context.Context, ref Reference, p string, opts BuildOptions) (*Build, error) { + localRef, ok := ref.(LocalReference) if !ok { return nil, fmt.Errorf("expected local chart reference") } @@ -53,14 +54,14 @@ func (b *localChartBuilder) Build(ctx context.Context, ref ChartReference, p str return nil, err } - // Load the chart metadata from the LocalChartReference to ensure it points + // Load the chart metadata from the LocalReference to ensure it points // to a chart curMeta, err := LoadChartMetadata(localRef.Path) if err != nil { return nil, err } - result := &ChartBuild{} + result := &Build{} result.Name = curMeta.Name // Set build specific metadata if instructed @@ -101,7 +102,7 @@ func (b *localChartBuilder) Build(ctx context.Context, ref ChartReference, p str // Merge chart values, if instructed var mergedValues map[string]interface{} if len(opts.GetValueFiles()) > 0 { - if mergedValues, err = mergeFileValues(localRef.BaseDir, opts.ValueFiles); err != nil { + if mergedValues, err = mergeFileValues(localRef.WorkDir, opts.ValueFiles); err != nil { return nil, fmt.Errorf("failed to merge value files: %w", err) } } diff --git a/internal/helm/chart_builder_local_test.go b/internal/helm/chart/builder_local_test.go similarity index 96% rename from internal/helm/chart_builder_local_test.go rename to internal/helm/chart/builder_local_test.go index c2f16d694..477d24890 100644 --- a/internal/helm/chart_builder_local_test.go +++ b/internal/helm/chart/builder_local_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helm +package chart import ( "os" @@ -99,16 +99,16 @@ func Test_copyFileToPath(t *testing.T) { }{ { name: "copies input file", - in: "testdata/local-index.yaml", + in: "../testdata/local-index.yaml", }, { name: "invalid input file", - in: "testdata/invalid.tgz", + in: "../testdata/invalid.tgz", wantErr: "failed to open file to copy from", }, { name: "invalid input directory", - in: "testdata/charts", + in: "../testdata/charts", wantErr: "failed to read from source during copy", }, } diff --git a/internal/helm/chart_builder_remote.go b/internal/helm/chart/builder_remote.go similarity index 91% rename from internal/helm/chart_builder_remote.go rename to internal/helm/chart/builder_remote.go index 18ff317d8..ce1953655 100644 --- a/internal/helm/chart_builder_remote.go +++ b/internal/helm/chart/builder_remote.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helm +package chart import ( "context" @@ -24,28 +24,31 @@ import ( "path/filepath" "github.com/Masterminds/semver/v3" - "github.com/fluxcd/pkg/runtime/transform" - "github.com/fluxcd/source-controller/internal/fs" helmchart "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/chartutil" "sigs.k8s.io/yaml" + + "github.com/fluxcd/pkg/runtime/transform" + + "github.com/fluxcd/source-controller/internal/fs" + "github.com/fluxcd/source-controller/internal/helm/repository" ) type remoteChartBuilder struct { - remote *ChartRepository + remote *repository.ChartRepository } -// NewRemoteChartBuilder returns a ChartBuilder capable of building a Helm -// chart with a RemoteChartReference from the given ChartRepository. -func NewRemoteChartBuilder(repository *ChartRepository) ChartBuilder { +// NewRemoteBuilder returns a Builder capable of building a Helm +// chart with a RemoteReference from the given Index. +func NewRemoteBuilder(repository *repository.ChartRepository) Builder { return &remoteChartBuilder{ remote: repository, } } -func (b *remoteChartBuilder) Build(_ context.Context, ref ChartReference, p string, opts BuildOptions) (*ChartBuild, error) { - remoteRef, ok := ref.(RemoteChartReference) +func (b *remoteChartBuilder) Build(_ context.Context, ref Reference, p string, opts BuildOptions) (*Build, error) { + remoteRef, ok := ref.(RemoteReference) if !ok { return nil, fmt.Errorf("expected remote chart reference") } @@ -59,13 +62,13 @@ func (b *remoteChartBuilder) Build(_ context.Context, ref ChartReference, p stri } defer b.remote.Unload() - // Get the current version for the RemoteChartReference + // Get the current version for the RemoteReference cv, err := b.remote.Get(remoteRef.Name, remoteRef.Version) if err != nil { return nil, fmt.Errorf("failed to get chart version for remote reference: %w", err) } - result := &ChartBuild{} + result := &Build{} result.Name = cv.Name result.Version = cv.Version // Set build specific metadata if instructed diff --git a/internal/helm/chart_builder_remote_test.go b/internal/helm/chart/builder_remote_test.go similarity index 92% rename from internal/helm/chart_builder_remote_test.go rename to internal/helm/chart/builder_remote_test.go index 260bcbce1..b7a2dae2f 100644 --- a/internal/helm/chart_builder_remote_test.go +++ b/internal/helm/chart/builder_remote_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helm +package chart import ( "testing" @@ -104,9 +104,9 @@ func Test_pathIsDir(t *testing.T) { p string want bool }{ - {name: "directory", p: "testdata/", want: true}, - {name: "file", p: "testdata/local-index.yaml", want: false}, - {name: "not found error", p: "testdata/does-not-exist.yaml", want: false}, + {name: "directory", p: "../testdata/", want: true}, + {name: "file", p: "../testdata/local-index.yaml", want: false}, + {name: "not found error", p: "../testdata/does-not-exist.yaml", want: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/helm/chart_builder_test.go b/internal/helm/chart/builder_test.go similarity index 89% rename from internal/helm/chart_builder_test.go rename to internal/helm/chart/builder_test.go index a4252be8f..92aec74f1 100644 --- a/internal/helm/chart_builder_test.go +++ b/internal/helm/chart/builder_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helm +package chart import ( "encoding/hex" @@ -30,18 +30,18 @@ import ( func TestChartBuildResult_String(t *testing.T) { g := NewWithT(t) - var result *ChartBuild + var result *Build g.Expect(result.String()).To(Equal("")) - result = &ChartBuild{} + result = &Build{} g.Expect(result.String()).To(Equal("")) - result = &ChartBuild{Path: "/foo/"} + result = &Build{Path: "/foo/"} g.Expect(result.String()).To(Equal("/foo/")) } func Test_packageToPath(t *testing.T) { g := NewWithT(t) - chart, err := loader.Load("testdata/charts/helmchart-0.1.0.tgz") + chart, err := loader.Load("../testdata/charts/helmchart-0.1.0.tgz") g.Expect(err).ToNot(HaveOccurred()) g.Expect(chart).ToNot(BeNil()) diff --git a/internal/helm/dependency_manager.go b/internal/helm/chart/dependency_manager.go similarity index 81% rename from internal/helm/dependency_manager.go rename to internal/helm/chart/dependency_manager.go index b8cd78571..2fa1df32c 100644 --- a/internal/helm/dependency_manager.go +++ b/internal/helm/chart/dependency_manager.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helm +package chart import ( "context" @@ -31,18 +31,20 @@ import ( "golang.org/x/sync/semaphore" helmchart "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" + + "github.com/fluxcd/source-controller/internal/helm/repository" ) -// GetChartRepositoryCallback must return a ChartRepository for the URL, -// or an error describing why it could not be returned. -type GetChartRepositoryCallback func(url string) (*ChartRepository, error) +// GetChartRepositoryCallback must return a repository.ChartRepository for the +// URL, or an error describing why it could not be returned. +type GetChartRepositoryCallback func(url string) (*repository.ChartRepository, error) // DependencyManager manages dependencies for a Helm chart. type DependencyManager struct { - // repositories contains a map of ChartRepository indexed by their + // repositories contains a map of Index indexed by their // normalized URL. It is used as a lookup table for missing // dependencies. - repositories map[string]*ChartRepository + repositories map[string]*repository.ChartRepository // getRepositoryCallback can be set to an on-demand GetChartRepositoryCallback // which returned result is cached to repositories. @@ -56,11 +58,12 @@ type DependencyManager struct { mu sync.Mutex } +// DependencyManagerOption configures an option on a DependencyManager. type DependencyManagerOption interface { applyToDependencyManager(dm *DependencyManager) } -type WithRepositories map[string]*ChartRepository +type WithRepositories map[string]*repository.ChartRepository func (o WithRepositories) applyToDependencyManager(dm *DependencyManager) { dm.repositories = o @@ -98,9 +101,9 @@ func (dm *DependencyManager) Clear() []error { } // Build compiles a set of missing dependencies from chart.Chart, and attempts to -// resolve and build them using the information from ChartReference. +// resolve and build them using the information from Reference. // It returns the number of resolved local and remote dependencies, or an error. -func (dm *DependencyManager) Build(ctx context.Context, ref ChartReference, chart *helmchart.Chart) (int, error) { +func (dm *DependencyManager) Build(ctx context.Context, ref Reference, chart *helmchart.Chart) (int, error) { // Collect dependency metadata var ( deps = chart.Dependencies() @@ -132,9 +135,9 @@ type chartWithLock struct { // build adds the given list of deps to the chart with the configured number of // concurrent workers. If the chart.Chart references a local dependency but no -// LocalChartReference is given, or any dependency could not be added, an error +// LocalReference is given, or any dependency could not be added, an error // is returned. The first error it encounters cancels all other workers. -func (dm *DependencyManager) build(ctx context.Context, ref ChartReference, chart *helmchart.Chart, deps map[string]*helmchart.Dependency) error { +func (dm *DependencyManager) build(ctx context.Context, ref Reference, c *helmchart.Chart, deps map[string]*helmchart.Dependency) error { current := dm.concurrent if current <= 0 { current = 1 @@ -143,7 +146,7 @@ func (dm *DependencyManager) build(ctx context.Context, ref ChartReference, char group, groupCtx := errgroup.WithContext(ctx) group.Go(func() error { sem := semaphore.NewWeighted(current) - chart := &chartWithLock{Chart: chart} + c := &chartWithLock{Chart: c} for name, dep := range deps { name, dep := name, dep if err := sem.Acquire(groupCtx, 1); err != nil { @@ -152,17 +155,17 @@ func (dm *DependencyManager) build(ctx context.Context, ref ChartReference, char group.Go(func() (err error) { defer sem.Release(1) if isLocalDep(dep) { - localRef, ok := ref.(LocalChartReference) + localRef, ok := ref.(LocalReference) if !ok { err = fmt.Errorf("failed to add local dependency '%s': no local chart reference", name) return } - if err = dm.addLocalDependency(localRef, chart, dep); err != nil { + if err = dm.addLocalDependency(localRef, c, dep); err != nil { err = fmt.Errorf("failed to add local dependency '%s': %w", name, err) } return } - if err = dm.addRemoteDependency(chart, dep); err != nil { + if err = dm.addRemoteDependency(c, dep); err != nil { err = fmt.Errorf("failed to add remote dependency '%s': %w", name, err) } return @@ -175,7 +178,7 @@ func (dm *DependencyManager) build(ctx context.Context, ref ChartReference, char // addLocalDependency attempts to resolve and add the given local chart.Dependency // to the chart. -func (dm *DependencyManager) addLocalDependency(ref LocalChartReference, chart *chartWithLock, dep *helmchart.Dependency) error { +func (dm *DependencyManager) addLocalDependency(ref LocalReference, c *chartWithLock, dep *helmchart.Dependency) error { sLocalChartPath, err := dm.secureLocalChartPath(ref, dep) if err != nil { return err @@ -197,7 +200,7 @@ func (dm *DependencyManager) addLocalDependency(ref LocalChartReference, chart * ch, err := loader.Load(sLocalChartPath) if err != nil { return fmt.Errorf("failed to load chart from '%s' (reference '%s'): %w", - strings.TrimPrefix(sLocalChartPath, ref.BaseDir), dep.Repository, err) + strings.TrimPrefix(sLocalChartPath, ref.WorkDir), dep.Repository, err) } ver, err := semver.NewVersion(ch.Metadata.Version) @@ -210,9 +213,9 @@ func (dm *DependencyManager) addLocalDependency(ref LocalChartReference, chart * return err } - chart.mu.Lock() - chart.AddDependency(ch) - chart.mu.Unlock() + c.mu.Lock() + c.AddDependency(ch) + c.mu.Unlock() return nil } @@ -249,19 +252,19 @@ func (dm *DependencyManager) addRemoteDependency(chart *chartWithLock, dep *helm } // resolveRepository first attempts to resolve the url from the repositories, falling back -// to getRepositoryCallback if set. It returns the resolved ChartRepository, or an error. -func (dm *DependencyManager) resolveRepository(url string) (_ *ChartRepository, err error) { +// to getRepositoryCallback if set. It returns the resolved Index, or an error. +func (dm *DependencyManager) resolveRepository(url string) (_ *repository.ChartRepository, err error) { dm.mu.Lock() defer dm.mu.Unlock() - nUrl := NormalizeChartRepositoryURL(url) + nUrl := repository.NormalizeURL(url) if _, ok := dm.repositories[nUrl]; !ok { if dm.getRepositoryCallback == nil { err = fmt.Errorf("no chart repository for URL '%s'", nUrl) return } if dm.repositories == nil { - dm.repositories = map[string]*ChartRepository{} + dm.repositories = map[string]*repository.ChartRepository{} } if dm.repositories[nUrl], err = dm.getRepositoryCallback(nUrl); err != nil { err = fmt.Errorf("failed to get chart repository for URL '%s': %w", nUrl, err) @@ -273,8 +276,8 @@ func (dm *DependencyManager) resolveRepository(url string) (_ *ChartRepository, // secureLocalChartPath returns the secure absolute path of a local dependency. // It does not allow the dependency's path to be outside the scope of -// LocalChartReference.BaseDir. -func (dm *DependencyManager) secureLocalChartPath(ref LocalChartReference, dep *helmchart.Dependency) (string, error) { +// LocalReference.WorkDir. +func (dm *DependencyManager) secureLocalChartPath(ref LocalReference, dep *helmchart.Dependency) (string, error) { localUrl, err := url.Parse(dep.Repository) if err != nil { return "", fmt.Errorf("failed to parse alleged local chart reference: %w", err) @@ -282,11 +285,11 @@ func (dm *DependencyManager) secureLocalChartPath(ref LocalChartReference, dep * if localUrl.Scheme != "" && localUrl.Scheme != "file" { return "", fmt.Errorf("'%s' is not a local chart reference", dep.Repository) } - relPath, err := filepath.Rel(ref.BaseDir, ref.Path) + relPath, err := filepath.Rel(ref.WorkDir, ref.Path) if err != nil { - return "", err + relPath = ref.Path } - return securejoin.SecureJoin(ref.BaseDir, filepath.Join(relPath, localUrl.Host, localUrl.Path)) + return securejoin.SecureJoin(ref.WorkDir, filepath.Join(relPath, localUrl.Host, localUrl.Path)) } // collectMissing returns a map with reqs that are missing from current, diff --git a/internal/helm/dependency_manager_test.go b/internal/helm/chart/dependency_manager_test.go similarity index 84% rename from internal/helm/dependency_manager_test.go rename to internal/helm/chart/dependency_manager_test.go index 388eff1f4..825fb3b1a 100644 --- a/internal/helm/dependency_manager_test.go +++ b/internal/helm/chart/dependency_manager_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helm +package chart import ( "context" @@ -29,26 +29,9 @@ import ( helmchart "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/repo" -) -var ( - // helmPackageFile contains the path to a Helm package in the v2 format - // without any dependencies - helmPackageFile = "testdata/charts/helmchart-0.1.0.tgz" - chartName = "helmchart" - chartVersion = "0.1.0" - chartLocalRepository = "file://../helmchart" - remoteDepFixture = helmchart.Dependency{ - Name: chartName, - Version: chartVersion, - Repository: "https://example.com/charts", - } - // helmPackageV1File contains the path to a Helm package in the v1 format, - // including dependencies in a requirements.yaml file which should be - // loaded - helmPackageV1File = "testdata/charts/helmchartwithdeps-v1-0.3.0.tgz" - chartNameV1 = "helmchartwithdeps-v1" - chartVersionV1 = "0.3.0" + "github.com/fluxcd/source-controller/internal/helm/getter" + "github.com/fluxcd/source-controller/internal/helm/repository" ) func TestDependencyManager_Build(t *testing.T) { @@ -56,7 +39,7 @@ func TestDependencyManager_Build(t *testing.T) { name string baseDir string path string - repositories map[string]*ChartRepository + repositories map[string]*repository.ChartRepository getChartRepositoryCallback GetChartRepositoryCallback want int wantChartFunc func(g *WithT, c *helmchart.Chart) @@ -70,13 +53,13 @@ func TestDependencyManager_Build(t *testing.T) { //}, { name: "build failure returns error", - baseDir: "testdata/charts", + baseDir: "./../testdata/charts", path: "helmchartwithdeps", wantErr: "failed to add remote dependency 'grafana': no chart repository for URL", }, { name: "no dependencies returns zero", - baseDir: "testdata/charts", + baseDir: "./../testdata/charts", path: "helmchart", want: 0, }, @@ -91,7 +74,7 @@ func TestDependencyManager_Build(t *testing.T) { got, err := NewDependencyManager( WithRepositories(tt.repositories), WithRepositoryCallback(tt.getChartRepositoryCallback), - ).Build(context.TODO(), LocalChartReference{BaseDir: tt.baseDir, Path: tt.path}, chart) + ).Build(context.TODO(), LocalReference{WorkDir: tt.baseDir, Path: tt.path}, chart) if tt.wantErr != "" { g.Expect(err).To(HaveOccurred()) @@ -135,7 +118,7 @@ func TestDependencyManager_build(t *testing.T) { g := NewWithT(t) dm := NewDependencyManager() - err := dm.build(context.TODO(), LocalChartReference{}, &helmchart.Chart{}, tt.deps) + err := dm.build(context.TODO(), LocalReference{}, &helmchart.Chart{}, tt.deps) if tt.wantErr != "" { g.Expect(err).To(HaveOccurred()) return @@ -180,7 +163,7 @@ func TestDependencyManager_addLocalDependency(t *testing.T) { Version: chartVersion, Repository: "file://../../../absolutely/invalid", }, - wantErr: "no chart found at 'testdata/charts/absolutely/invalid'", + wantErr: "no chart found at '../testdata/charts/absolutely/invalid'", }, { name: "invalid chart archive", @@ -207,7 +190,7 @@ func TestDependencyManager_addLocalDependency(t *testing.T) { dm := NewDependencyManager() chart := &helmchart.Chart{} - err := dm.addLocalDependency(LocalChartReference{BaseDir: "testdata/charts", Path: "helmchartwithdeps"}, + err := dm.addLocalDependency(LocalReference{WorkDir: "../testdata/charts", Path: "helmchartwithdeps"}, &chartWithLock{Chart: chart}, tt.dep) if tt.wantErr != "" { g.Expect(err).To(HaveOccurred()) @@ -222,23 +205,23 @@ func TestDependencyManager_addLocalDependency(t *testing.T) { func TestDependencyManager_addRemoteDependency(t *testing.T) { g := NewWithT(t) - chartB, err := os.ReadFile("testdata/charts/helmchart-0.1.0.tgz") + chartB, err := os.ReadFile("../testdata/charts/helmchart-0.1.0.tgz") g.Expect(err).ToNot(HaveOccurred()) g.Expect(chartB).ToNot(BeEmpty()) tests := []struct { name string - repositories map[string]*ChartRepository + repositories map[string]*repository.ChartRepository dep *helmchart.Dependency wantFunc func(g *WithT, c *helmchart.Chart) wantErr string }{ { name: "adds remote dependency", - repositories: map[string]*ChartRepository{ + repositories: map[string]*repository.ChartRepository{ "https://example.com/": { - Client: &mockGetter{ - response: chartB, + Client: &getter.MockGetter{ + Response: chartB, }, Index: &repo.IndexFile{ Entries: map[string]repo.ChartVersions{ @@ -266,7 +249,7 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) { }, { name: "resolve repository error", - repositories: map[string]*ChartRepository{}, + repositories: map[string]*repository.ChartRepository{}, dep: &helmchart.Dependency{ Repository: "https://example.com", }, @@ -274,7 +257,7 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) { }, { name: "strategic load error", - repositories: map[string]*ChartRepository{ + repositories: map[string]*repository.ChartRepository{ "https://example.com/": { CachePath: "/invalid/cache/path/foo", RWMutex: &sync.RWMutex{}, @@ -287,7 +270,7 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) { }, { name: "repository get error", - repositories: map[string]*ChartRepository{ + repositories: map[string]*repository.ChartRepository{ "https://example.com/": { Index: &repo.IndexFile{}, RWMutex: &sync.RWMutex{}, @@ -300,7 +283,7 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) { }, { name: "repository version constraint error", - repositories: map[string]*ChartRepository{ + repositories: map[string]*repository.ChartRepository{ "https://example.com/": { Index: &repo.IndexFile{ Entries: map[string]repo.ChartVersions{ @@ -326,7 +309,7 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) { }, { name: "repository chart download error", - repositories: map[string]*ChartRepository{ + repositories: map[string]*repository.ChartRepository{ "https://example.com/": { Index: &repo.IndexFile{ Entries: map[string]repo.ChartVersions{ @@ -352,9 +335,9 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) { }, { name: "chart load error", - repositories: map[string]*ChartRepository{ + repositories: map[string]*repository.ChartRepository{ "https://example.com/": { - Client: &mockGetter{}, + Client: &getter.MockGetter{}, Index: &repo.IndexFile{ Entries: map[string]repo.ChartVersions{ chartName: { @@ -404,40 +387,40 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) { func TestDependencyManager_resolveRepository(t *testing.T) { tests := []struct { name string - repositories map[string]*ChartRepository + repositories map[string]*repository.ChartRepository getChartRepositoryCallback GetChartRepositoryCallback url string - want *ChartRepository - wantRepositories map[string]*ChartRepository + want *repository.ChartRepository + wantRepositories map[string]*repository.ChartRepository wantErr string }{ { name: "resolves from repositories index", url: "https://example.com", - repositories: map[string]*ChartRepository{ + repositories: map[string]*repository.ChartRepository{ "https://example.com/": {URL: "https://example.com"}, }, - want: &ChartRepository{URL: "https://example.com"}, + want: &repository.ChartRepository{URL: "https://example.com"}, }, { name: "resolves from callback", url: "https://example.com", - getChartRepositoryCallback: func(url string) (*ChartRepository, error) { - return &ChartRepository{URL: "https://example.com"}, nil + getChartRepositoryCallback: func(url string) (*repository.ChartRepository, error) { + return &repository.ChartRepository{URL: "https://example.com"}, nil }, - want: &ChartRepository{URL: "https://example.com"}, - wantRepositories: map[string]*ChartRepository{ + want: &repository.ChartRepository{URL: "https://example.com"}, + wantRepositories: map[string]*repository.ChartRepository{ "https://example.com/": {URL: "https://example.com"}, }, }, { name: "error from callback", url: "https://example.com", - getChartRepositoryCallback: func(url string) (*ChartRepository, error) { + getChartRepositoryCallback: func(url string) (*repository.ChartRepository, error) { return nil, errors.New("a very unique error") }, wantErr: "a very unique error", - wantRepositories: map[string]*ChartRepository{}, + wantRepositories: map[string]*repository.ChartRepository{}, }, { name: "error on not found", @@ -518,7 +501,7 @@ func TestDependencyManager_secureLocalChartPath(t *testing.T) { g := NewWithT(t) dm := NewDependencyManager() - got, err := dm.secureLocalChartPath(LocalChartReference{BaseDir: tt.baseDir, Path: tt.path}, tt.dep) + got, err := dm.secureLocalChartPath(LocalReference{WorkDir: tt.baseDir, Path: tt.path}, tt.dep) if tt.wantErr != "" { g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) diff --git a/internal/helm/chart.go b/internal/helm/chart/metadata.go similarity index 96% rename from internal/helm/chart.go rename to internal/helm/chart/metadata.go index 4f89cab61..24e452089 100644 --- a/internal/helm/chart.go +++ b/internal/helm/chart/metadata.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helm +package chart import ( "archive/tar" @@ -33,6 +33,8 @@ import ( helmchart "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chartutil" "sigs.k8s.io/yaml" + + "github.com/fluxcd/source-controller/internal/helm" ) // OverwriteChartDefaultValues overwrites the chart default values file with the given data. @@ -115,8 +117,8 @@ func LoadChartMetadataFromDir(dir string) (*helmchart.Metadata, error) { if stat.IsDir() { return nil, fmt.Errorf("'%s' is a directory", stat.Name()) } - if stat.Size() > MaxChartFileSize { - return nil, fmt.Errorf("size of '%s' exceeds '%d' limit", stat.Name(), MaxChartFileSize) + if stat.Size() > helm.MaxChartFileSize { + return nil, fmt.Errorf("size of '%s' exceeds '%d' limit", stat.Name(), helm.MaxChartFileSize) } } @@ -142,8 +144,8 @@ func LoadChartMetadataFromArchive(archive string) (*helmchart.Metadata, error) { } return nil, err } - if stat.Size() > MaxChartSize { - return nil, fmt.Errorf("size of chart '%s' exceeds '%d' limit", stat.Name(), MaxChartSize) + if stat.Size() > helm.MaxChartSize { + return nil, fmt.Errorf("size of chart '%s' exceeds '%d' limit", stat.Name(), helm.MaxChartSize) } f, err := os.Open(archive) diff --git a/internal/helm/chart_test.go b/internal/helm/chart/metadata_test.go similarity index 85% rename from internal/helm/chart_test.go rename to internal/helm/chart/metadata_test.go index ac7114e87..f2294ff6b 100644 --- a/internal/helm/chart_test.go +++ b/internal/helm/chart/metadata_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helm +package chart import ( "testing" @@ -25,6 +25,19 @@ import ( ) var ( + // helmPackageFile contains the path to a Helm package in the v2 format + // without any dependencies + helmPackageFile = "../testdata/charts/helmchart-0.1.0.tgz" + chartName = "helmchart" + chartVersion = "0.1.0" + + // helmPackageV1File contains the path to a Helm package in the v1 format, + // including dependencies in a requirements.yaml file which should be + // loaded + helmPackageV1File = "../testdata/charts/helmchartwithdeps-v1-0.3.0.tgz" + chartNameV1 = "helmchartwithdeps-v1" + chartVersionV1 = "0.3.0" + originalValuesFixture = []byte(`override: original `) chartFilesFixture = []*helmchart.File{ @@ -123,21 +136,21 @@ func TestLoadChartMetadataFromDir(t *testing.T) { }{ { name: "Loads from dir", - dir: "testdata/charts/helmchart", + dir: "../testdata/charts/helmchart", wantName: "helmchart", wantVersion: "0.1.0", }, { name: "Loads from v1 dir including requirements.yaml", - dir: "testdata/charts/helmchartwithdeps-v1", + dir: "../testdata/charts/helmchartwithdeps-v1", wantName: chartNameV1, wantVersion: chartVersionV1, wantDependencyCount: 1, }, { name: "Error if no Chart.yaml", - dir: "testdata/charts/", - wantErr: "testdata/charts/Chart.yaml: no such file or directory", + dir: "../testdata/charts/", + wantErr: "../testdata/charts/Chart.yaml: no such file or directory", }, } for _, tt := range tests { @@ -186,12 +199,12 @@ func TestLoadChartMetadataFromArchive(t *testing.T) { }, { name: "Error on not found", - archive: "testdata/invalid.tgz", + archive: "../testdata/invalid.tgz", wantErr: "no such file or directory", }, { name: "Error if no Chart.yaml", - archive: "testdata/charts/empty.tgz", + archive: "../testdata/charts/empty.tgz", wantErr: "no 'Chart.yaml' found", }, } diff --git a/internal/helm/getter.go b/internal/helm/getter/getter.go similarity index 99% rename from internal/helm/getter.go rename to internal/helm/getter/getter.go index 1ca8b0e9b..583bac5f7 100644 --- a/internal/helm/getter.go +++ b/internal/helm/getter/getter.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helm +package getter import ( "fmt" diff --git a/internal/helm/getter_test.go b/internal/helm/getter/getter_test.go similarity index 99% rename from internal/helm/getter_test.go rename to internal/helm/getter/getter_test.go index 2c55e7cbb..6437e5b35 100644 --- a/internal/helm/getter_test.go +++ b/internal/helm/getter/getter_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helm +package getter import ( "os" diff --git a/internal/helm/getter/mock.go b/internal/helm/getter/mock.go new file mode 100644 index 000000000..91cd2b7bc --- /dev/null +++ b/internal/helm/getter/mock.go @@ -0,0 +1,41 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package getter + +import ( + "bytes" + + "helm.sh/helm/v3/pkg/getter" +) + +// MockGetter can be used as a simple mocking getter.Getter implementation. +type MockGetter struct { + Response []byte + + requestedURL string +} + +func (g *MockGetter) Get(u string, _ ...getter.Option) (*bytes.Buffer, error) { + g.requestedURL = u + r := g.Response + return bytes.NewBuffer(r), nil +} + +// LastGet returns the last requested URL for Get. +func (g *MockGetter) LastGet() string { + return g.requestedURL +} diff --git a/internal/helm/repository.go b/internal/helm/repository/chart_repository.go similarity index 98% rename from internal/helm/repository.go rename to internal/helm/repository/chart_repository.go index eb9e668a1..638355f80 100644 --- a/internal/helm/repository.go +++ b/internal/helm/repository/chart_repository.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helm +package repository import ( "bytes" @@ -36,6 +36,8 @@ import ( "sigs.k8s.io/yaml" "github.com/fluxcd/pkg/version" + + "github.com/fluxcd/source-controller/internal/helm" ) var ErrNoChartIndex = errors.New("no chart index") @@ -241,8 +243,8 @@ func (r *ChartRepository) LoadFromFile(path string) error { } return err } - if stat.Size() > MaxIndexSize { - return fmt.Errorf("size of index '%s' exceeds '%d' limit", stat.Name(), MaxIndexSize) + if stat.Size() > helm.MaxIndexSize { + return fmt.Errorf("size of index '%s' exceeds '%d' limit", stat.Name(), helm.MaxIndexSize) } b, err := os.ReadFile(path) if err != nil { @@ -350,7 +352,7 @@ func (r *ChartRepository) HasCacheFile() bool { } // Unload can be used to signal the Go garbage collector the Index can -// be freed from memory if the ChartRepository object is expected to +// be freed from memory if the Index object is expected to // continue to exist in the stack for some time. func (r *ChartRepository) Unload() { if r == nil { diff --git a/internal/helm/repository_test.go b/internal/helm/repository/chart_repository_test.go similarity index 93% rename from internal/helm/repository_test.go rename to internal/helm/repository/chart_repository_test.go index 9c124b791..b6f191f3b 100644 --- a/internal/helm/repository_test.go +++ b/internal/helm/repository/chart_repository_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helm +package repository import ( "bytes" @@ -27,39 +27,29 @@ import ( . "github.com/onsi/gomega" "helm.sh/helm/v3/pkg/chart" - "helm.sh/helm/v3/pkg/getter" + helmgetter "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/repo" + + "github.com/fluxcd/source-controller/internal/helm/getter" ) var now = time.Now() const ( - testFile = "testdata/local-index.yaml" - chartmuseumTestFile = "testdata/chartmuseum-index.yaml" - unorderedTestFile = "testdata/local-index-unordered.yaml" + testFile = "../testdata/local-index.yaml" + chartmuseumTestFile = "../testdata/chartmuseum-index.yaml" + unorderedTestFile = "../testdata/local-index-unordered.yaml" ) -// mockGetter can be used as a simple mocking getter.Getter implementation. -type mockGetter struct { - requestedURL string - response []byte -} - -func (g *mockGetter) Get(url string, _ ...getter.Option) (*bytes.Buffer, error) { - g.requestedURL = url - r := g.response - return bytes.NewBuffer(r), nil -} - func TestNewChartRepository(t *testing.T) { repositoryURL := "https://example.com" - providers := getter.Providers{ - getter.Provider{ + providers := helmgetter.Providers{ + helmgetter.Provider{ Schemes: []string{"https"}, - New: getter.NewHTTPGetter, + New: helmgetter.NewHTTPGetter, }, } - options := []getter.Option{getter.WithBasicAuth("username", "password")} + options := []helmgetter.Option{helmgetter.WithBasicAuth("username", "password")} t.Run("should construct chart repository", func(t *testing.T) { g := NewWithT(t) @@ -230,7 +220,7 @@ func TestChartRepository_DownloadChart(t *testing.T) { g := NewWithT(t) t.Parallel() - mg := mockGetter{} + mg := getter.MockGetter{} r := &ChartRepository{ URL: tt.url, Client: &mg, @@ -241,7 +231,7 @@ func TestChartRepository_DownloadChart(t *testing.T) { g.Expect(res).To(BeNil()) return } - g.Expect(mg.requestedURL).To(Equal(tt.wantURL)) + g.Expect(mg.LastGet()).To(Equal(tt.wantURL)) g.Expect(res).ToNot(BeNil()) g.Expect(err).ToNot(HaveOccurred()) }) @@ -254,7 +244,7 @@ func TestChartRepository_DownloadIndex(t *testing.T) { b, err := os.ReadFile(chartmuseumTestFile) g.Expect(err).ToNot(HaveOccurred()) - mg := mockGetter{response: b} + mg := getter.MockGetter{Response: b} r := &ChartRepository{ URL: "https://example.com", Client: &mg, @@ -263,7 +253,7 @@ func TestChartRepository_DownloadIndex(t *testing.T) { buf := bytes.NewBuffer([]byte{}) g.Expect(r.DownloadIndex(buf)).To(Succeed()) g.Expect(buf.Bytes()).To(Equal(b)) - g.Expect(mg.requestedURL).To(Equal(r.URL + "/index.yaml")) + g.Expect(mg.LastGet()).To(Equal(r.URL + "/index.yaml")) g.Expect(err).To(BeNil()) } @@ -384,8 +374,8 @@ func TestChartRepository_LoadIndexFromFile(t *testing.T) { func TestChartRepository_CacheIndex(t *testing.T) { g := NewWithT(t) - mg := mockGetter{response: []byte("foo")} - expectSum := fmt.Sprintf("%x", sha256.Sum256(mg.response)) + mg := getter.MockGetter{Response: []byte("foo")} + expectSum := fmt.Sprintf("%x", sha256.Sum256(mg.Response)) r := newChartRepository() r.URL = "https://example.com" @@ -399,7 +389,7 @@ func TestChartRepository_CacheIndex(t *testing.T) { g.Expect(r.CachePath).To(BeARegularFile()) b, _ := os.ReadFile(r.CachePath) - g.Expect(b).To(Equal(mg.response)) + g.Expect(b).To(Equal(mg.Response)) g.Expect(sum).To(BeEquivalentTo(expectSum)) } diff --git a/internal/helm/utils.go b/internal/helm/repository/utils.go similarity index 77% rename from internal/helm/utils.go rename to internal/helm/repository/utils.go index ff2221c61..b02b13782 100644 --- a/internal/helm/utils.go +++ b/internal/helm/repository/utils.go @@ -1,5 +1,5 @@ /* -Copyright 2020 The Flux authors +Copyright 2021 The Flux authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,12 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package helm +package repository import "strings" -// NormalizeChartRepositoryURL ensures repository urls are normalized -func NormalizeChartRepositoryURL(url string) string { +// NormalizeURL normalizes a ChartRepository URL by ensuring it ends with a +// single "/". +func NormalizeURL(url string) string { if url != "" { return strings.TrimRight(url, "/") + "/" } diff --git a/internal/helm/repository/utils_test.go b/internal/helm/repository/utils_test.go new file mode 100644 index 000000000..fe4cf80ee --- /dev/null +++ b/internal/helm/repository/utils_test.go @@ -0,0 +1,44 @@ +package repository + +import ( + "testing" + + . "github.com/onsi/gomega" +) + +func TestNormalizeURL(t *testing.T) { + tests := []struct { + name string + url string + want string + }{ + { + name: "with slash", + url: "http://example.com/", + want: "http://example.com/", + }, + { + name: "without slash", + url: "http://example.com", + want: "http://example.com/", + }, + { + name: "double slash", + url: "http://example.com//", + want: "http://example.com/", + }, + { + name: "empty", + url: "", + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + got := NormalizeURL(tt.url) + g.Expect(got).To(Equal(tt.want)) + }) + } +} diff --git a/internal/helm/utils_test.go b/internal/helm/utils_test.go deleted file mode 100644 index 62a9e92c2..000000000 --- a/internal/helm/utils_test.go +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright 2021 The Flux authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package helm - -import ( - "testing" - - . "github.com/onsi/gomega" -) - -func TestNormalizeChartRepositoryURL(t *testing.T) { - tests := []struct { - name string - url string - want string - }{ - { - name: "with slash", - url: "http://example.com/", - want: "http://example.com/", - }, - { - name: "without slash", - url: "http://example.com", - want: "http://example.com/", - }, - { - name: "double slash", - url: "http://example.com//", - want: "http://example.com/", - }, - { - name: "empty", - url: "", - want: "", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - g := NewWithT(t) - - got := NormalizeChartRepositoryURL(tt.url) - g.Expect(got).To(Equal(tt.want)) - }) - } -} From 32e19ebcd0e1a75b08c752dd616eb1b4a742dbed Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Tue, 16 Nov 2021 09:50:07 +0100 Subject: [PATCH 09/23] controllers: more tidying of wiring Dealing with some loose ends around making observations, and code style. The loaded byes of a chart are used as a revision to ensure e.g. periodic builds with unstable ordering of items do not trigger a false positive. Signed-off-by: Hidde Beydals --- controllers/helmchart_controller.go | 84 +++++++++--------------- controllers/helmchart_controller_test.go | 24 ------- controllers/helmrepository_controller.go | 38 +++++------ 3 files changed, 46 insertions(+), 100 deletions(-) diff --git a/controllers/helmchart_controller.go b/controllers/helmchart_controller.go index d31f6c2bb..3c1be0a7d 100644 --- a/controllers/helmchart_controller.go +++ b/controllers/helmchart_controller.go @@ -22,13 +22,12 @@ import ( "net/url" "os" "path/filepath" - "regexp" "strings" "time" securejoin "github.com/cyphar/filepath-securejoin" "github.com/go-logr/logr" - extgetter "helm.sh/helm/v3/pkg/getter" + helmgetter "helm.sh/helm/v3/pkg/getter" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" @@ -69,7 +68,7 @@ type HelmChartReconciler struct { client.Client Scheme *runtime.Scheme Storage *Storage - Getters extgetter.Providers + Getters helmgetter.Providers EventRecorder kuberecorder.EventRecorder ExternalEventRecorder *events.Recorder MetricsRecorder *metrics.Recorder @@ -199,7 +198,7 @@ func (r *HelmChartReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } // Create working directory - workDir, err := os.MkdirTemp("", chart.Kind + "-" + chart.Namespace + "-" + chart.Name + "-") + workDir, err := os.MkdirTemp("", chart.Kind+"-"+chart.Namespace+"-"+chart.Name+"-") if err != nil { err = fmt.Errorf("failed to create temporary working directory: %w", err) chart = sourcev1.HelmChartNotReady(*chart.DeepCopy(), sourcev1.ChartPullFailedReason, err.Error()) @@ -216,21 +215,6 @@ func (r *HelmChartReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( var reconcileErr error switch typedSource := source.(type) { case *sourcev1.HelmRepository: - // TODO: move this to a validation webhook once the discussion around - // certificates has settled: https://github.com/fluxcd/image-reflector-controller/issues/69 - if err := validHelmChartName(chart.Spec.Chart); err != nil { - reconciledChart = sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()) - log.Error(err, "validation failed") - if err := r.updateStatus(ctx, req, reconciledChart.Status); err != nil { - log.Info(fmt.Sprintf("%v", reconciledChart.Status)) - log.Error(err, "unable to update status") - return ctrl.Result{Requeue: true}, err - } - r.event(ctx, reconciledChart, events.EventSeverityError, err.Error()) - r.recordReadiness(ctx, reconciledChart) - // Do not requeue as there is no chance on recovery. - return ctrl.Result{Requeue: false}, nil - } reconciledChart, reconcileErr = r.fromHelmRepository(ctx, *typedSource, *chart.DeepCopy(), workDir, changed) case *sourcev1.GitRepository, *sourcev1.Bucket: reconciledChart, reconcileErr = r.fromTarballArtifact(ctx, *typedSource.GetArtifact(), *chart.DeepCopy(), @@ -309,10 +293,10 @@ func (r *HelmChartReconciler) getSource(ctx context.Context, chart sourcev1.Helm func (r *HelmChartReconciler) fromHelmRepository(ctx context.Context, repo sourcev1.HelmRepository, c sourcev1.HelmChart, workDir string, force bool) (sourcev1.HelmChart, error) { // Configure Index getter options - clientOpts := []extgetter.Option{ - extgetter.WithURL(repo.Spec.URL), - extgetter.WithTimeout(repo.Spec.Timeout.Duration), - extgetter.WithPassCredentialsAll(repo.Spec.PassCredentials), + clientOpts := []helmgetter.Option{ + helmgetter.WithURL(repo.Spec.URL), + helmgetter.WithTimeout(repo.Spec.Timeout.Duration), + helmgetter.WithPassCredentialsAll(repo.Spec.PassCredentials), } if secret, err := r.getHelmRepositorySecret(ctx, &repo); err != nil { return sourcev1.HelmChartNotReady(c, sourcev1.AuthenticationFailedReason, err.Error()), err @@ -423,7 +407,7 @@ func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source so err = fmt.Errorf("artifact untar error: %w", err) return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } - if err =f.Close(); err != nil { + if err = f.Close(); err != nil { err = fmt.Errorf("artifact close error: %w", err) return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } @@ -440,20 +424,17 @@ func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source so return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } dm := chart.NewDependencyManager( - chart.WithRepositoryCallback(r.getNamespacedChartRepositoryCallback(ctx, authDir, c.GetNamespace())), + chart.WithRepositoryCallback(r.namespacedChartRepositoryCallback(ctx, authDir, c.GetNamespace())), ) defer dm.Clear() - // Get any cached chart - var cachedChart string - if artifact := c.Status.Artifact; artifact != nil { - cachedChart = artifact.Path - } - + // Configure builder options, including any previously cached chart buildsOpts := chart.BuildOptions{ - ValueFiles: c.GetValuesFiles(), - CachedChart: cachedChart, - Force: force, + ValueFiles: c.GetValuesFiles(), + Force: force, + } + if artifact := c.Status.Artifact; artifact != nil { + buildsOpts.CachedChart = artifact.Path } // Add revision metadata to chart build @@ -465,7 +446,7 @@ func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source so // Build chart chartB := chart.NewLocalBuilder(dm) - build, err := chartB.Build(ctx, chart.LocalReference{BaseDir: sourceDir, Path: chartPath}, filepath.Join(workDir, "chart.tgz"), buildsOpts) + build, err := chartB.Build(ctx, chart.LocalReference{WorkDir: sourceDir, Path: chartPath}, filepath.Join(workDir, "chart.tgz"), buildsOpts) if err != nil { return sourcev1.HelmChartNotReady(c, sourcev1.ChartPackageFailedReason, err.Error()), err } @@ -475,7 +456,8 @@ func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source so // If the path of the returned build equals the cache path, // there are no changes to the chart - if build.Path == cachedChart { + if apimeta.IsStatusConditionTrue(c.Status.Conditions, meta.ReadyCondition) && + build.Path == buildsOpts.CachedChart { // Ensure hostname is updated if c.GetArtifact().URL != newArtifact.URL { r.Storage.SetArtifactURL(c.GetArtifact()) @@ -515,11 +497,17 @@ func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source so return sourcev1.HelmChartReady(c, newArtifact, cUrl, sourcev1.ChartPackageSucceededReason, build.Summary()), nil } -// TODO(hidde): factor out to helper? -func (r *HelmChartReconciler) getNamespacedChartRepositoryCallback(ctx context.Context, dir, namespace string) chart.GetChartRepositoryCallback { +// namespacedChartRepositoryCallback returns a chart.GetChartRepositoryCallback +// scoped to the given namespace. Credentials for retrieved v1beta1.HelmRepository +// objects are stored in the given directory. +// The returned callback returns a repository.ChartRepository configured with the +// retrieved v1beta1.HelmRepository, or a shim with defaults if no object could +// be found. +func (r *HelmChartReconciler) namespacedChartRepositoryCallback(ctx context.Context, dir, namespace string) chart.GetChartRepositoryCallback { return func(url string) (*repository.ChartRepository, error) { repo, err := r.resolveDependencyRepository(ctx, url, namespace) if err != nil { + // Return Kubernetes client errors, but ignore others if errors.ReasonForError(err) != metav1.StatusReasonUnknown { return nil, err } @@ -530,10 +518,10 @@ func (r *HelmChartReconciler) getNamespacedChartRepositoryCallback(ctx context.C }, } } - clientOpts := []extgetter.Option{ - extgetter.WithURL(repo.Spec.URL), - extgetter.WithTimeout(repo.Spec.Timeout.Duration), - extgetter.WithPassCredentialsAll(repo.Spec.PassCredentials), + clientOpts := []helmgetter.Option{ + helmgetter.WithURL(repo.Spec.URL), + helmgetter.WithTimeout(repo.Spec.Timeout.Duration), + helmgetter.WithPassCredentialsAll(repo.Spec.PassCredentials), } if secret, err := r.getHelmRepositorySecret(ctx, repo); err != nil { return nil, err @@ -801,18 +789,6 @@ func (r *HelmChartReconciler) requestsForBucketChange(o client.Object) []reconci return reqs } -// validHelmChartName returns an error if the given string is not a -// valid Helm chart name; a valid name must be lower case letters -// and numbers, words may be separated with dashes (-). -// Ref: https://helm.sh/docs/chart_best_practices/conventions/#chart-names -func validHelmChartName(s string) error { - chartFmt := regexp.MustCompile("^([-a-z0-9]*)$") - if !chartFmt.MatchString(s) { - return fmt.Errorf("invalid chart name %q, a valid name must be lower case letters and numbers and MAY be separated with dashes (-)", s) - } - return nil -} - func (r *HelmChartReconciler) recordSuspension(ctx context.Context, chart sourcev1.HelmChart) { if r.MetricsRecorder == nil { return diff --git a/controllers/helmchart_controller_test.go b/controllers/helmchart_controller_test.go index ceb30842f..82df1bc35 100644 --- a/controllers/helmchart_controller_test.go +++ b/controllers/helmchart_controller_test.go @@ -25,7 +25,6 @@ import ( "path" "path/filepath" "strings" - "testing" "time" "github.com/fluxcd/pkg/apis/meta" @@ -1327,26 +1326,3 @@ var _ = Describe("HelmChartReconciler", func() { }) }) }) - -func Test_validHelmChartName(t *testing.T) { - tests := []struct { - name string - chart string - expectErr bool - }{ - {"valid", "drupal", false}, - {"valid dash", "nginx-lego", false}, - {"valid dashes", "aws-cluster-autoscaler", false}, - {"valid alphanum", "ng1nx-leg0", false}, - {"invalid slash", "artifactory/invalid", true}, - {"invalid dot", "in.valid", true}, - {"invalid uppercase", "inValid", true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := validHelmChartName(tt.chart); (err != nil) != tt.expectErr { - t.Errorf("validHelmChartName() error = %v, expectErr %v", err, tt.expectErr) - } - }) - } -} diff --git a/controllers/helmrepository_controller.go b/controllers/helmrepository_controller.go index 8ab87201d..5a29a7734 100644 --- a/controllers/helmrepository_controller.go +++ b/controllers/helmrepository_controller.go @@ -24,7 +24,7 @@ import ( "time" "github.com/go-logr/logr" - extgetter "helm.sh/helm/v3/pkg/getter" + helmgetter "helm.sh/helm/v3/pkg/getter" corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -43,9 +43,9 @@ import ( "github.com/fluxcd/pkg/runtime/metrics" "github.com/fluxcd/pkg/runtime/predicates" + sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" "github.com/fluxcd/source-controller/internal/helm/getter" "github.com/fluxcd/source-controller/internal/helm/repository" - sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" ) // +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=helmrepositories,verbs=get;list;watch;create;update;patch;delete @@ -58,7 +58,7 @@ type HelmRepositoryReconciler struct { client.Client Scheme *runtime.Scheme Storage *Storage - Getters extgetter.Providers + Getters helmgetter.Providers EventRecorder kuberecorder.EventRecorder ExternalEventRecorder *events.Recorder MetricsRecorder *metrics.Recorder @@ -171,10 +171,10 @@ func (r *HelmRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reque } func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repo sourcev1.HelmRepository) (sourcev1.HelmRepository, error) { - clientOpts := []extgetter.Option{ - extgetter.WithURL(repo.Spec.URL), - extgetter.WithTimeout(repo.Spec.Timeout.Duration), - extgetter.WithPassCredentialsAll(repo.Spec.PassCredentials), + clientOpts := []helmgetter.Option{ + helmgetter.WithURL(repo.Spec.URL), + helmgetter.WithTimeout(repo.Spec.Timeout.Duration), + helmgetter.WithPassCredentialsAll(repo.Spec.PassCredentials), } if repo.Spec.SecretRef != nil { name := types.NamespacedName{ @@ -189,7 +189,7 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repo sourcev1. return sourcev1.HelmRepositoryNotReady(repo, sourcev1.AuthenticationFailedReason, err.Error()), err } - authDir, err := os.MkdirTemp("", "helm-repository-") + authDir, err := os.MkdirTemp("", repo.Kind+"-"+repo.Namespace+"-"+repo.Name+"-") if err != nil { err = fmt.Errorf("failed to create temporary working directory for credentials: %w", err) return sourcev1.HelmRepositoryNotReady(repo, sourcev1.AuthenticationFailedReason, err.Error()), err @@ -213,7 +213,7 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repo sourcev1. return sourcev1.HelmRepositoryNotReady(repo, sourcev1.IndexationFailedReason, err.Error()), err } } - revision, err := chartRepo.CacheIndex() + checksum, err := chartRepo.CacheIndex() if err != nil { err = fmt.Errorf("failed to download repository index: %w", err) return sourcev1.HelmRepositoryNotReady(repo, sourcev1.IndexationFailedReason, err.Error()), err @@ -222,12 +222,12 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repo sourcev1. artifact := r.Storage.NewArtifactFor(repo.Kind, repo.ObjectMeta.GetObjectMeta(), - revision, - fmt.Sprintf("index-%s.yaml", revision)) + "", + fmt.Sprintf("index-%s.yaml", checksum)) // Return early on unchanged index if apimeta.IsStatusConditionTrue(repo.Status.Conditions, meta.ReadyCondition) && - repo.GetArtifact().HasRevision(artifact.Revision) { + (repo.GetArtifact() != nil && repo.GetArtifact().Checksum == checksum) { if artifact.URL != repo.GetArtifact().URL { r.Storage.SetArtifactURL(repo.GetArtifact()) repo.Status.URL = r.Storage.SetHostname(repo.Status.URL) @@ -239,7 +239,9 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repo sourcev1. if err := chartRepo.LoadFromCache(); err != nil { return sourcev1.HelmRepositoryNotReady(repo, sourcev1.IndexationFailedReason, err.Error()), err } - defer chartRepo.Unload() + // The repository checksum is the SHA256 of the loaded bytes, after sorting + artifact.Revision = chartRepo.Checksum + chartRepo.Unload() // Create artifact dir err = r.Storage.MkdirAll(artifact) @@ -257,17 +259,9 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repo sourcev1. defer unlock() // Save artifact to storage - storageTarget := r.Storage.LocalPath(artifact) - if storageTarget == "" { - err := fmt.Errorf("failed to calcalute local storage path to store artifact to") - return sourcev1.HelmRepositoryNotReady(repo, sourcev1.StorageOperationFailedReason, err.Error()), err - } - if err = chartRepo.Index.WriteFile(storageTarget, 0644); err != nil { + if err = r.Storage.CopyFromPath(&artifact, chartRepo.CachePath); err != nil { return sourcev1.HelmRepositoryNotReady(repo, sourcev1.StorageOperationFailedReason, err.Error()), err } - // TODO(hidde): it would be better to make the Storage deal with this - artifact.Checksum = chartRepo.Checksum - artifact.LastUpdateTime = metav1.Now() // Update index symlink indexURL, err := r.Storage.Symlink(artifact, "index.yaml") From 7c910e37a2bed1debcaa70dde942d4dc4884ca6d Mon Sep 17 00:00:00 2001 From: Sunny Date: Tue, 16 Nov 2021 16:26:05 +0530 Subject: [PATCH 10/23] internal/helm: local builder & dep manager test Add more chart local builder and dependency manager tests. Signed-off-by: Sunny --- go.mod | 1 + go.sum | 7 + internal/helm/chart/builder_local_test.go | 211 ++++++++++++++++++ .../helm/chart/dependency_manager_test.go | 84 ++++++- 4 files changed, 294 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index c4503b710..5246fc455 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/minio/minio-go/v7 v7.0.10 github.com/onsi/ginkgo v1.16.4 github.com/onsi/gomega v1.14.0 + github.com/otiai10/copy v1.7.0 github.com/spf13/pflag v1.0.5 github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940 // indirect github.com/yvasiyarov/gorelic v0.0.7 // indirect diff --git a/go.sum b/go.sum index 593aa3e0b..a252cf16f 100644 --- a/go.sum +++ b/go.sum @@ -738,6 +738,13 @@ github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxS github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/otiai10/copy v1.7.0 h1:hVoPiN+t+7d2nzzwMiDHPSOogsWAStewq3TwU05+clE= +github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= +github.com/otiai10/mint v1.3.3 h1:7JgpsBaN0uMkyju4tbYHu0mnM55hNKVYLsXmwr15NQI= +github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= diff --git a/internal/helm/chart/builder_local_test.go b/internal/helm/chart/builder_local_test.go index 477d24890..1e0acb744 100644 --- a/internal/helm/chart/builder_local_test.go +++ b/internal/helm/chart/builder_local_test.go @@ -17,14 +17,225 @@ limitations under the License. package chart import ( + "context" "os" "path/filepath" + "sync" "testing" . "github.com/onsi/gomega" + "github.com/otiai10/copy" helmchart "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/chartutil" + "helm.sh/helm/v3/pkg/repo" + + "github.com/fluxcd/source-controller/internal/helm/getter" + "github.com/fluxcd/source-controller/internal/helm/repository" ) +func TestLocalBuilder_Build(t *testing.T) { + g := NewWithT(t) + + // Prepare chart repositories to be used for charts with remote dependency. + chartB, err := os.ReadFile("./../testdata/charts/helmchart-0.1.0.tgz") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(chartB).ToNot(BeEmpty()) + mockRepo := func() *repository.ChartRepository { + return &repository.ChartRepository{ + Client: &getter.MockGetter{ + Response: chartB, + }, + Index: &repo.IndexFile{ + Entries: map[string]repo.ChartVersions{ + "grafana": { + &repo.ChartVersion{ + Metadata: &helmchart.Metadata{ + Name: "grafana", + Version: "6.17.4", + }, + URLs: []string{"https://example.com/grafana.tgz"}, + }, + }, + }, + }, + RWMutex: &sync.RWMutex{}, + } + } + + tests := []struct { + name string + reference Reference + buildOpts BuildOptions + valueFiles []helmchart.File + repositories map[string]*repository.ChartRepository + dependentChartPaths []string + wantValues chartutil.Values + wantVersion string + wantPackaged bool + wantErr string + }{ + { + name: "invalid reference", + reference: RemoteReference{}, + wantErr: "expected local chart reference", + }, + { + name: "invalid local reference - no path", + reference: LocalReference{}, + wantErr: "no path set for local chart reference", + }, + { + name: "invalid local reference - no file", + reference: LocalReference{Path: "/tmp/non-existent-path.xyz"}, + wantErr: "no such file or directory", + }, + { + name: "invalid version metadata", + reference: LocalReference{Path: "./../testdata/charts/helmchart"}, + buildOpts: BuildOptions{VersionMetadata: "^"}, + wantErr: "Invalid Metadata string", + }, + { + name: "with version metadata", + reference: LocalReference{Path: "./../testdata/charts/helmchart"}, + buildOpts: BuildOptions{VersionMetadata: "foo"}, + wantVersion: "0.1.0+foo", + wantPackaged: true, + }, + // TODO: Test setting BuildOptions CachedChart and Force. + { + name: "already packaged chart", + reference: LocalReference{Path: "./../testdata/charts/helmchart-0.1.0.tgz"}, + wantVersion: "0.1.0", + wantPackaged: false, + }, + { + name: "default values", + reference: LocalReference{Path: "./../testdata/charts/helmchart"}, + wantValues: chartutil.Values{ + "replicaCount": float64(1), + }, + wantVersion: "0.1.0", + wantPackaged: true, + }, + { + name: "with value files", + reference: LocalReference{Path: "./../testdata/charts/helmchart"}, + buildOpts: BuildOptions{ + ValueFiles: []string{"custom-values1.yaml", "custom-values2.yaml"}, + }, + valueFiles: []helmchart.File{ + { + Name: "custom-values1.yaml", + Data: []byte(`replicaCount: 11 +nameOverride: "foo-name-override"`), + }, + { + Name: "custom-values2.yaml", + Data: []byte(`replicaCount: 20 +fullnameOverride: "full-foo-name-override"`), + }, + }, + wantValues: chartutil.Values{ + "replicaCount": float64(20), + "nameOverride": "foo-name-override", + "fullnameOverride": "full-foo-name-override", + }, + wantVersion: "0.1.0", + wantPackaged: true, + }, + { + name: "chart with dependencies", + reference: LocalReference{Path: "./../testdata/charts/helmchartwithdeps"}, + repositories: map[string]*repository.ChartRepository{ + "https://grafana.github.io/helm-charts/": mockRepo(), + }, + dependentChartPaths: []string{"./../testdata/charts/helmchart"}, + wantVersion: "0.1.0", + wantPackaged: true, + }, + { + name: "v1 chart", + reference: LocalReference{Path: "./../testdata/charts/helmchart-v1"}, + wantValues: chartutil.Values{ + "replicaCount": float64(1), + }, + wantVersion: "0.2.0", + wantPackaged: true, + }, + { + name: "v1 chart with dependencies", + reference: LocalReference{Path: "./../testdata/charts/helmchartwithdeps-v1"}, + repositories: map[string]*repository.ChartRepository{ + "https://grafana.github.io/helm-charts/": mockRepo(), + }, + dependentChartPaths: []string{"./../testdata/charts/helmchart-v1"}, + wantVersion: "0.3.0", + wantPackaged: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + workDir, err := os.MkdirTemp("", "local-builder-") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(workDir) + + // Only if the reference is a LocalReference, set the WorkDir. + localRef, ok := tt.reference.(LocalReference) + if ok { + localRef.WorkDir = workDir + tt.reference = localRef + } + + // Write value file in the base dir. + for _, f := range tt.valueFiles { + vPath := filepath.Join(workDir, f.Name) + g.Expect(os.WriteFile(vPath, f.Data, 0644)).ToNot(HaveOccurred()) + } + + // Write chart dependencies in the base dir. + for _, dcp := range tt.dependentChartPaths { + // Construct the chart path relative to the testdata chart. + helmchartDir := filepath.Join(workDir, "testdata", "charts", filepath.Base(dcp)) + g.Expect(copy.Copy(dcp, helmchartDir)).ToNot(HaveOccurred()) + } + + // Target path with name similar to the workDir. + targetPath := workDir + ".tgz" + defer os.RemoveAll(targetPath) + + dm := NewDependencyManager( + WithRepositories(tt.repositories), + ) + + b := NewLocalBuilder(dm) + cb, err := b.Build(context.TODO(), tt.reference, targetPath, tt.buildOpts) + + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(cb).To(BeZero()) + return + } + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(cb.Packaged).To(Equal(tt.wantPackaged), "unexpected Build.Packaged value") + g.Expect(cb.Path).ToNot(BeEmpty(), "empty Build.Path") + + // Load the resulting chart and verify the values. + resultChart, err := loader.Load(cb.Path) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(resultChart.Metadata.Version).To(Equal(tt.wantVersion)) + + for k, v := range tt.wantValues { + g.Expect(v).To(Equal(resultChart.Values[k])) + } + }) + } +} + func Test_mergeFileValues(t *testing.T) { tests := []struct { name string diff --git a/internal/helm/chart/dependency_manager_test.go b/internal/helm/chart/dependency_manager_test.go index 825fb3b1a..da4b70a67 100644 --- a/internal/helm/chart/dependency_manager_test.go +++ b/internal/helm/chart/dependency_manager_test.go @@ -35,6 +35,36 @@ import ( ) func TestDependencyManager_Build(t *testing.T) { + g := NewWithT(t) + + // Mock chart used as grafana chart in the test below. The cached repository + // takes care of the actual grafana related details in the chart index. + chartGrafana, err := os.ReadFile("./../testdata/charts/helmchart-0.1.0.tgz") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(chartGrafana).ToNot(BeEmpty()) + + mockRepo := func() *repository.ChartRepository { + return &repository.ChartRepository{ + Client: &getter.MockGetter{ + Response: chartGrafana, + }, + Index: &repo.IndexFile{ + Entries: map[string]repo.ChartVersions{ + "grafana": { + &repo.ChartVersion{ + Metadata: &helmchart.Metadata{ + Name: "grafana", + Version: "6.17.4", + }, + URLs: []string{"https://example.com/grafana.tgz"}, + }, + }, + }, + }, + RWMutex: &sync.RWMutex{}, + } + } + tests := []struct { name string baseDir string @@ -45,12 +75,6 @@ func TestDependencyManager_Build(t *testing.T) { wantChartFunc func(g *WithT, c *helmchart.Chart) wantErr string }{ - //{ - // // TODO(hidde): add various happy paths - //}, - //{ - // // TODO(hidde): test Chart.lock - //}, { name: "build failure returns error", baseDir: "./../testdata/charts", @@ -61,7 +85,44 @@ func TestDependencyManager_Build(t *testing.T) { name: "no dependencies returns zero", baseDir: "./../testdata/charts", path: "helmchart", - want: 0, + wantChartFunc: func(g *WithT, c *helmchart.Chart) { + g.Expect(c.Dependencies()).To(HaveLen(0)) + }, + want: 0, + }, + { + name: "no dependency returns zero - v1", + baseDir: "./../testdata/charts", + path: "helmchart-v1", + wantChartFunc: func(g *WithT, c *helmchart.Chart) { + g.Expect(c.Dependencies()).To(HaveLen(0)) + }, + want: 0, + }, + { + name: "build with dependencies using lock file", + baseDir: "./../testdata/charts", + path: "helmchartwithdeps", + repositories: map[string]*repository.ChartRepository{ + "https://grafana.github.io/helm-charts/": mockRepo(), + }, + getChartRepositoryCallback: func(url string) (*repository.ChartRepository, error) { + return &repository.ChartRepository{URL: "https://grafana.github.io/helm-charts/"}, nil + }, + wantChartFunc: func(g *WithT, c *helmchart.Chart) { + g.Expect(c.Dependencies()).To(HaveLen(2)) + g.Expect(c.Lock.Dependencies).To(HaveLen(3)) + }, + want: 2, + }, + { + name: "build with dependencies - v1", + baseDir: "./../testdata/charts", + path: "helmchartwithdeps-v1", + wantChartFunc: func(g *WithT, c *helmchart.Chart) { + g.Expect(c.Dependencies()).To(HaveLen(1)) + }, + want: 1, }, } for _, tt := range tests { @@ -71,10 +132,11 @@ func TestDependencyManager_Build(t *testing.T) { chart, err := loader.Load(filepath.Join(tt.baseDir, tt.path)) g.Expect(err).ToNot(HaveOccurred()) - got, err := NewDependencyManager( + dm := NewDependencyManager( WithRepositories(tt.repositories), WithRepositoryCallback(tt.getChartRepositoryCallback), - ).Build(context.TODO(), LocalReference{WorkDir: tt.baseDir, Path: tt.path}, chart) + ) + got, err := dm.Build(context.TODO(), LocalReference{WorkDir: tt.baseDir, Path: tt.path}, chart) if tt.wantErr != "" { g.Expect(err).To(HaveOccurred()) @@ -198,6 +260,10 @@ func TestDependencyManager_addLocalDependency(t *testing.T) { return } g.Expect(err).ToNot(HaveOccurred()) + + if tt.wantFunc != nil { + tt.wantFunc(g, chart) + } }) } } From 753abed30cf25ab901c3f895f460d80779f520e2 Mon Sep 17 00:00:00 2001 From: Sunny Date: Tue, 16 Nov 2021 20:23:52 +0530 Subject: [PATCH 11/23] internal/helm: add remote builder tests - For remote builds, if the build option has a version metadata, the chart should be repackaged with the provided version. - Update internal/helm/testdata/charts/helmchart-0.1.0.tgz to include value files for testing merge chart values. Signed-off-by: Sunny --- internal/helm/chart/builder_remote.go | 7 +- internal/helm/chart/builder_remote_test.go | 187 ++++++++++++++++++ .../helm/testdata/charts/helmchart-0.1.0.tgz | Bin 3277 -> 3354 bytes 3 files changed, 192 insertions(+), 2 deletions(-) diff --git a/internal/helm/chart/builder_remote.go b/internal/helm/chart/builder_remote.go index ce1953655..2caceb39c 100644 --- a/internal/helm/chart/builder_remote.go +++ b/internal/helm/chart/builder_remote.go @@ -100,8 +100,9 @@ func (b *remoteChartBuilder) Build(_ context.Context, ref Reference, p string, o return nil, fmt.Errorf("failed to download chart for remote reference: %w", err) } - // Use literal chart copy from remote if no custom value files options are set - if len(opts.GetValueFiles()) == 0 { + // Use literal chart copy from remote if no custom value files options are + // set or build option version metadata isn't set. + if len(opts.GetValueFiles()) == 0 && opts.VersionMetadata == "" { if err = validatePackageAndWriteToPath(res, p); err != nil { return nil, err } @@ -127,6 +128,8 @@ func (b *remoteChartBuilder) Build(_ context.Context, ref Reference, p string, o result.ValueFiles = opts.GetValueFiles() } + chart.Metadata.Version = result.Version + // Package the chart with the custom values if err = packageToPath(chart, p); err != nil { return nil, err diff --git a/internal/helm/chart/builder_remote_test.go b/internal/helm/chart/builder_remote_test.go index b7a2dae2f..431ac0a6c 100644 --- a/internal/helm/chart/builder_remote_test.go +++ b/internal/helm/chart/builder_remote_test.go @@ -17,13 +17,200 @@ limitations under the License. package chart import ( + "bytes" + "context" + "math/rand" + "os" + "strings" + "sync" "testing" + "time" . "github.com/onsi/gomega" helmchart "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/chartutil" + helmgetter "helm.sh/helm/v3/pkg/getter" + + "github.com/fluxcd/source-controller/internal/helm/repository" ) +var letterRunes = []rune("abcdefghijklmnopqrstuvwxyz1234567890") + +func randStringRunes(n int) string { + b := make([]rune, n) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return string(b) +} + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +// mockIndexChartGetter returns specific response for index and chart queries. +type mockIndexChartGetter struct { + IndexResponse []byte + ChartResponse []byte + requestedURL string +} + +func (g *mockIndexChartGetter) Get(u string, _ ...helmgetter.Option) (*bytes.Buffer, error) { + g.requestedURL = u + r := g.ChartResponse + if strings.HasSuffix(u, "index.yaml") { + r = g.IndexResponse + } + return bytes.NewBuffer(r), nil +} + +func (g *mockIndexChartGetter) LastGet() string { + return g.requestedURL +} + +func TestRemoteBuilder_Build(t *testing.T) { + g := NewWithT(t) + + chartGrafana, err := os.ReadFile("./../testdata/charts/helmchart-0.1.0.tgz") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(chartGrafana).ToNot(BeEmpty()) + + index := []byte(` +apiVersion: v1 +entries: + grafana: + - urls: + - https://example.com/grafana.tgz + description: string + version: 6.17.4 +`) + + mockGetter := &mockIndexChartGetter{ + IndexResponse: index, + ChartResponse: chartGrafana, + } + + mockRepo := func() *repository.ChartRepository { + return &repository.ChartRepository{ + URL: "https://grafana.github.io/helm-charts/", + Client: mockGetter, + RWMutex: &sync.RWMutex{}, + } + } + + tests := []struct { + name string + reference Reference + buildOpts BuildOptions + repository *repository.ChartRepository + wantValues chartutil.Values + wantVersion string + wantPackaged bool + wantErr string + }{ + { + name: "invalid reference", + reference: LocalReference{}, + wantErr: "expected remote chart reference", + }, + { + name: "invalid reference - no name", + reference: RemoteReference{}, + wantErr: "no name set for remote chart reference", + }, + { + name: "chart not in repo", + reference: RemoteReference{Name: "foo"}, + repository: mockRepo(), + wantErr: "failed to get chart version for remote reference", + }, + { + name: "chart version not in repo", + reference: RemoteReference{Name: "grafana", Version: "1.1.1"}, + repository: mockRepo(), + wantErr: "failed to get chart version for remote reference", + }, + { + name: "invalid version metadata", + reference: RemoteReference{Name: "grafana"}, + repository: mockRepo(), + buildOpts: BuildOptions{VersionMetadata: "^"}, + wantErr: "Invalid Metadata string", + }, + { + name: "with version metadata", + reference: RemoteReference{Name: "grafana"}, + repository: mockRepo(), + buildOpts: BuildOptions{VersionMetadata: "foo"}, + wantVersion: "6.17.4+foo", + wantPackaged: true, + }, + // TODO: Test setting BuildOptions CachedChart and Force. + { + name: "default values", + reference: RemoteReference{Name: "grafana"}, + repository: mockRepo(), + wantVersion: "0.1.0", + wantValues: chartutil.Values{ + "replicaCount": float64(1), + }, + }, + { + name: "merge values", + reference: RemoteReference{Name: "grafana"}, + buildOpts: BuildOptions{ + ValueFiles: []string{"a.yaml", "b.yaml", "c.yaml"}, + }, + repository: mockRepo(), + wantVersion: "6.17.4", + wantValues: chartutil.Values{ + "a": "b", + "b": "d", + }, + wantPackaged: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + targetPath := "/tmp/remote-chart-builder-" + randStringRunes(5) + ".tgz" + defer os.RemoveAll(targetPath) + + if tt.repository != nil { + _, err := tt.repository.CacheIndex() + g.Expect(err).ToNot(HaveOccurred()) + // Cleanup the cache index path. + defer os.Remove(tt.repository.CachePath) + } + + b := NewRemoteBuilder(tt.repository) + + cb, err := b.Build(context.TODO(), tt.reference, targetPath, tt.buildOpts) + + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + g.Expect(cb).To(BeZero()) + return + } + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(cb.Packaged).To(Equal(tt.wantPackaged), "unexpected Build.Packaged value") + g.Expect(cb.Path).ToNot(BeEmpty(), "empty Build.Path") + + // Load the resulting chart and verify the values. + resultChart, err := loader.Load(cb.Path) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(resultChart.Metadata.Version).To(Equal(tt.wantVersion)) + + for k, v := range tt.wantValues { + g.Expect(v).To(Equal(resultChart.Values[k])) + } + }) + } +} + func Test_mergeChartValues(t *testing.T) { tests := []struct { name string diff --git a/internal/helm/testdata/charts/helmchart-0.1.0.tgz b/internal/helm/testdata/charts/helmchart-0.1.0.tgz index f64a32eeeb54fc24a44390478a3adf5bcb5cf754..1ffdde531f6c9e4ee48963ba512a302970994d14 100644 GIT binary patch delta 3327 zcmV~>nKMg}uYx;qe}X4}UF48{|q8xflLnUd_pU zCqOd#1*M{bM=(9`NR}->_JZDV5PAu!Skg?}ckkeTk)>dF42*1&CdQG z1NPwzMA3tl8Dr!DNX&$yTF6-hd`$U` z2ax5A4Fsd{EQ0gVh0p_u3b`(Z4RXdVF_x%R1Ydu)o`2}$6s4pIM&SG2C}(W#rKeC% zX^ijU*h=96?89Y-F&$AO zAV#1kxJE<^$Q1?__k09gXi%tu@T8z+bZ)4<2I2@&1!LqW4M)focKVt<@Vs%NKyYrN4B%ompBNc|v1o<(pFhM`eKJ6cRKGNg=ZipnVnAdw;~pF8mG z!w35;kt82En|%^O{|{xTW=dmPQ;5v@J9QzrL1@1Ihe8=auMov5%5ZQIPVeGgzFl(w9W0?dcVMZ^x?kThc$FcA}GzX!;Op15rZpjr8}Z-y$nXD@8r zTp`fWOcffgnC&}yomO6+L?v<=qgt-rKM``36UJsBF&1fxJhA&s3s82O&mbv`e>vK2 zKz}WOA(k0*xxkCaohD1ot*naCj#a83BiOlre&?q;i< zeF(p)&+ON~$NyLHf9ajO7au>rzYO$^exMk0NBlnwkB=Jhzkl36*v0?HfL;JUpno%z zTBZ*F`1ApWGh?s3h%*YoSQX)kioE%}19Swz7nd`ZzO-<}uvl0ON%$DynQ>qcovdM7 z$7XN=D(t|zOZRr7bvBEf`?JvbG;!$9Nk$?WPRE2_ zm5~_12=NJ-afZRg^6g>)9Ra^6!Od(>{CQx!SYW8Cui?)DFN9e5W!j(&CCX!#CkTF( zpOwMiw{v~j5bk`?5X}%)_Y#n5Os#B66P>GoB#BuFwC{=!3*QP5PlR2%sDImiR)`Tj zTPkaV-aiPE{5N4HTFBdU4*iu8EE(E?=|rG}&9t4xaz+bym|DI@epR-t%hpF5iFpa(>#ffR@}g ze79i;olfcW!Jp6l`MfZ~7k`u|cmw5Hvzj7H3xx2cwXwh&nlmYX(Xs=osYFKD#=_d! zZZyVa=C_444nCfJyLk8G`-Z|Yk!-u+8Tcg2g4OKL3_NL9W~Ip2>*FX_+9>R#$mvvk2Jl^M>AFT*|ug_IC@INy_yI{J*Kr-<02==J@`e zown(DWe3)l)|gDu_b|(9m$%p}4vUc?vRjwf{`EIbsIG7KT(ofDH;|Yy6W;y*^uhn; z-^ZpznhNfuRDZWg=`b7_^EObEuKgz3kc<}73S9ozhM|h0$A^M+0JSadruuH#urlAR z=V8m1lSzBlhWrM<=0aoRWVlWxC7g^g=?rJlO8#_N6m?rQSC(wH&|Pz)omMGagYB+l zv}ItKj7Dq8U%TYRE^d1I_7e8(^Qu?aFE+WURkWa}e}8w11K)hU^er!LKhwP2M3L90 zS~`aYZNGR}J8QHOY4HG09m?+oZTVk<856S<`2$@CxGVqb^_%^_YJ#>e`I=4E8*>%fO1o}rTU@ioHEv~(uNtxJ z`hVTm;@5c88jBU1dbjaDoSr(y?O-#s<$q<9?V+xM-F5#f>>aGv|91C39|i8{@%hC-fmWt=&Kh{+T9ha*_mZQEF@{00taP`)e0e^Z!)TFn10IUze8;#sNs#RR)!CXds?v&G> z4p$ob!#r9#mR1||n=s3}P%8;{F{ak) zchIp3PQCup8UtbFij)~reZH-?4RC$lf@URyM)Y=N zbo6E|{`bRO{C^Bs`Ta*(`YFoVPh0nKmw#LCj8s=YYG>b0arnFCj!AKtA##j*&^#%& z9eh_#(W~{PdlY&4e@Q2Mi~Sd6{(nF22JW)|!-Ho3|4lFK@9h6EVCzoIZH}Tq-`Zc? zQamg@NBh8yPUE}p!#owV*}q1m-QP|uILFu=FaCfVxXb>J!^ZvZgYfWZ7ylmxTFrVO zk``O`w+a58#Rd2{BS)%4#S+Xgz_e3R24g0MuHj1g_%%pmM1RvM+L}l0J>iMB4;;tt7bY(aP9k#P@>X|H-cXTfUf#yXH^% zQB8TbA~YoNRi4?qkIH)$sO!vo6%5Ih_bSk7=DqrVZy&x8Nky*U{CMoW@`6l?Uoh5Q zKocZghbP4^UNBX$NU-}Ir6864lRxGEe%RZ>{=;Vc?;RY5yZZm50EuAu9c|YgJTO%D zKm2Ri|L*^PcoY~$5bx~&-xn(TkN;Bk-`k!49|eXHBs=@x!47t?gMUBxR{#J2|Nm$% JpX&fp002eojw1j7 delta 3250 zcmV;j3{CTz8qFDyJAXTSZ`-(&{ac@6PU#PQTrJCawGfa4dP%PrY?~TQw~Iwl6tpz9 z*-)gGq@4I%-)BFNlq^}66F1E!w}AOaBFn>>;c#A@p~uM5cub`39-F_xl%(v@9gi>! z!^8c3`#%iB=Krwwa{p0p@8$ksuNS`D+kX`H_Id~5BM9%Bk$<+4D^291@Rw;d7xxbt zB%_~EDk^vclU3m!OIz&9r^@8vYwu3RYuaC?x!r4^VQ{s60+mj9{Tq zo}OkH0m(8(V`8RzlM;0p^n%cP=(TUhqeqqfPYBCV{T&YA2K(QAxw~Tj`@P=Q{_kV# zzzGgX&NR6Fw|_>xP>R6Yflp(qfGQyH=j-?HIzu5-qBSNkqznzf3C4^_gb9(93>YfV z0tN^fQ3?~FTnm_rT!O|lV??70JWrx^3&$eoI)a|((UgoZ@&F`eLQyT`Gy*=N{K^B! za>n|C(Rdoc>F`YGzC?vw7t{JVW9Jx4)GC56U#%zlFn>WQX@U{>zBkMnTY2dzloJ}` z>o~StcmO+ao?%Rf6ct>KQIAmqfdUSQ0|Ye|IZMnEV~Ip#;z98gftER1M4!e8h9rm~ z=rOJ!(E@UXfrULC0v8$-Y9Kr*a2c5!Y%hQ~L{z~DIZ8tjaz**bA}-gq5U&RoKJvUw zBIe)Fr1UCT9*S{;25%daHte^}97eS6$t~AQizEK1r zbp)?M&!c=KQCX&t8?{R!7!sxoR>HZ^Hsq*cC4Yv}#UfM%RCJ5^LClC!m8Pu%YNk3Q zHlfWoO#2?dSSW2N8v&RPk&B2Oydr7FFkm7k%zh7$5j}Ql37}c|v}2|!tFvb|ZZ09{ zXr>AYSHyN4zD_GGPofgJj8QH2?hk}q<%F>*NQ_0AB2TQIX#vWr`4p1E_!qOS0%`#a z$$tbv8`~$w_G7_SOhxY4ipRp}vhfv01#^WmaE^j7C4h3H!U~mPmB;SC)XHo&2oA_6 zR0^IV*9tDF9s{FVvqB+CR$hS0+No`ur-DT+6qPBFF4thf8>JuQ~_SB`4a86Fo2o+HCp3u$-P(}Ed4;1vAJSI;BEA>~w0 z?bko!|Eu`F@Xp<{51-zg2l`6iQH;4E{_ll{2aWi@e|WIJjsN#CdI7veXDGEy9e;lR z_znhBW3Rl3GYY|272%1ByxFV+bO^y`mot{Wv~a{QpIZn?_z2;#abOS~tw3AHW?%s- z=)k&5_jasxHk%njkDGHfGcam41LM@@^QhaM%^WF<7v}TX41x-~1M`}XOrgOa3$6*} z3j8yXU|&e>FW4>OQDB0R!-rQU41dAdhm$wo`X4@iI(q!nc&Au17@>wv$An*%kr=@+ z@e!GEhQZn5?R*X$0pFG2X0=EDEHGXyFj&=>@N2*eCgy&bHYkIM@|fibf?wrlW$^dy zTAwwTJMA}EGlob{`jPM1PML!dfHu zcY-AUNtlTi@;ccAfN@U|oN7qP6g0U&ATY#BNGZ>Ck--{$uA_w0evy@u38BoWX;8#x zJN^_o#Dv2o(ec>jYj&L`d#l;Olb_yaHHo2fxzX-Dsdn%D&BxED$1NLZ(QQI^6N1p` zlujS~+038K3L|_+8c;1IGR!tM)~-TRO0?S0QslqexNtOl6*`Jws)Go|QkS|u}QLeNR*in<7k<(h^N>P)9 zXku~Kgz0~#it19Pz1^2V7=%GDdKJD3i<5*qV%2=ONmNToY$O09Mt^g>yCgD!$E7^M z(1>1zb=UdOL2EniY_#Qn-^R!?lq%5Kos@!Z%Kr{}dyV|>;9!4moB!R%n9aJ+VM5c$ zRv(6xq4BIosS!&&#t5Ewt$Xf1_ugD(_HfuJY!ro}0D}u!8u$TP<~)YOJ%9U1)AM{dq*vf~DrCkXnPU-Xc%4<3@|3fn<|lFt+?%0Jfz4H1w^7d@}WH{+e7lx!PH9QU4D3M7Z4q7zOOYt7qW0zmILL2*QK{V&0r=e*F*6Cr8@soegm7M`!kxh?s-WA zYg=nfrs#W^ZMEB5=!%14WPt42Ew+FCwF|1t+dUU89QXkeGiJiO|L;Ed-~8Lylt@#- z9hd4lE**qJV}ITTYTUKoL~D}KVp)O9|Joo_QS^9Mpbns>;%=$07Y!@(-D)1TXgQg* zXJyI{_$wD08z;k6EGgk+gh^*Gjh6DK^P;HRD!H;`vw`fI3+=RW;Tmi=C8I49%VacK zN&eb7FSNMn>Dx=#*RQKyVLw~vrk25iqW;|}4t%rO!hg3sz5Yz|Y#l~kUux+b8o2%9 zVdbpRiloH@Jaj0()o9EA63m#GrpWK;I>1f&UvIzJ|2y10JlN)c_c7}If0AXYJL!2B zlqV6KEVRHYkEK!BTH2_*;6dT}qQcZVZFAVdxBmc6d4gQSzL{;(r=lNdj|>r+E%${hsAbA7M0IN zqjOOXgOwKO5-VE#NmABMt`_s$gjIjNZmZiodr(KT+)r#}saV-+m$HplD3e8$wH0ess5D=4cL_%aW+ zXuQ0pQYd4sv!yR;GVGU))aCCpDYS?Mi{R67zxExY6Xd8=Uy1=PQ!`ZFVneL1?HNJ0 z`PPzcFeZ^C^l!vocI$51_^JuozU3=6S#Qi$U`y?~jcswo8ds>5J-%wjqU(2Ci(jEp zYkw?OWa{0<+faJw7`GkkM_c|^Hrej#D%ef;zrx<`O8swp|MPyv4V~OYmB*xZXGNvw zhRkmXx0Tux?cuRE| zOWkp~N@O|eOCc{P9|ae$Tp6G@Kuvnf2YN&7)d|bso%R#HUU;?dfnOpXa zUs_`zEM1W@Q>xF`_4WZ|&hQ4#mzXN=aTyx+0>KJvw)y7O6OcwUT)n_ht=8o$)qjNQP?yme(9-T|kV~>@m)S#mqV0Hy(H8$pCw))xKRh^ixf=g> zxAFfz#?tRU%F<6!)_&N!kGuTaa%ZHv`cXUkc8bH_HFr#k!wiul)cxj3vF+fybc$ZC zFWsWZv;Y6n$zEgsMVbHC-M~%uzkj#e?Ek;)^$xf8e;;GxPRwEImj2 z#I;W2AK!<0sL^Ks8kKf`JF(y#V{^Rt1vhY${U3&n``^3a-r+X>-^*w<>%K@@Y}wx= z_;(s=62Tx>(|>`uTJ#s} z_4Ww9JTah8zLr&1IZ_e8mnWO7|C&p#7NJC=Z@Mh=pWy`MLwc;r{sZ$!M_fp}V;VQ+ z|GS6H-+%5MY~%mEj2-9`tx(+S#|M}O@-;fc2c97paKCQmboLwbdYD|G$avj9GDHU+`$Ov4gn zDD4ln0xvi@|8}m0L~jR*8%dv!&mo~wdBKS4uKnxS_kzLy$gcfcz8H_X=1=)iO?bCL zG$8Rsp4q#P%6lHD%glQo49JD|JkV+8J^x>C2R;)?MXuoV1-7;7(}36idZ klj6G~2$#?bwd(_y>&t3jhHB|8^VVp#Vkz0Mi3q1^@s6 From dd3afce3be6dabff975c78074cd7f5ea48790a49 Mon Sep 17 00:00:00 2001 From: Sunny Date: Wed, 17 Nov 2021 03:56:16 +0530 Subject: [PATCH 12/23] internal/helm: add cached chart build tests Cached chart build tests for both local and remote builder. Signed-off-by: Sunny --- internal/helm/chart/builder_local_test.go | 39 +++++++++- internal/helm/chart/builder_remote_test.go | 90 +++++++++++++++++----- 2 files changed, 109 insertions(+), 20 deletions(-) diff --git a/internal/helm/chart/builder_local_test.go b/internal/helm/chart/builder_local_test.go index 1e0acb744..7f42ee905 100644 --- a/internal/helm/chart/builder_local_test.go +++ b/internal/helm/chart/builder_local_test.go @@ -103,7 +103,6 @@ func TestLocalBuilder_Build(t *testing.T) { wantVersion: "0.1.0+foo", wantPackaged: true, }, - // TODO: Test setting BuildOptions CachedChart and Force. { name: "already packaged chart", reference: LocalReference{Path: "./../testdata/charts/helmchart-0.1.0.tgz"}, @@ -236,6 +235,44 @@ fullnameOverride: "full-foo-name-override"`), } } +func TestLocalBuilder_Build_CachedChart(t *testing.T) { + g := NewWithT(t) + + workDir, err := os.MkdirTemp("", "local-builder-") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(workDir) + + reference := LocalReference{Path: "./../testdata/charts/helmchart"} + + dm := NewDependencyManager() + b := NewLocalBuilder(dm) + + tmpDir, err := os.MkdirTemp("", "local-chart-") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpDir) + + // Build first time. + targetPath := filepath.Join(tmpDir, "chart1.tgz") + buildOpts := BuildOptions{} + cb, err := b.Build(context.TODO(), reference, targetPath, buildOpts) + g.Expect(err).ToNot(HaveOccurred()) + + // Set the result as the CachedChart for second build. + buildOpts.CachedChart = cb.Path + + targetPath2 := filepath.Join(tmpDir, "chart2.tgz") + defer os.RemoveAll(targetPath2) + cb, err = b.Build(context.TODO(), reference, targetPath2, buildOpts) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(cb.Path).To(Equal(targetPath)) + + // Rebuild with build option Force. + buildOpts.Force = true + cb, err = b.Build(context.TODO(), reference, targetPath2, buildOpts) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(cb.Path).To(Equal(targetPath2)) +} + func Test_mergeFileValues(t *testing.T) { tests := []struct { name string diff --git a/internal/helm/chart/builder_remote_test.go b/internal/helm/chart/builder_remote_test.go index 431ac0a6c..e8ad6be54 100644 --- a/internal/helm/chart/builder_remote_test.go +++ b/internal/helm/chart/builder_remote_test.go @@ -19,12 +19,10 @@ package chart import ( "bytes" "context" - "math/rand" "os" "strings" "sync" "testing" - "time" . "github.com/onsi/gomega" helmchart "helm.sh/helm/v3/pkg/chart" @@ -35,20 +33,6 @@ import ( "github.com/fluxcd/source-controller/internal/helm/repository" ) -var letterRunes = []rune("abcdefghijklmnopqrstuvwxyz1234567890") - -func randStringRunes(n int) string { - b := make([]rune, n) - for i := range b { - b[i] = letterRunes[rand.Intn(len(letterRunes))] - } - return string(b) -} - -func init() { - rand.Seed(time.Now().UnixNano()) -} - // mockIndexChartGetter returns specific response for index and chart queries. type mockIndexChartGetter struct { IndexResponse []byte @@ -146,7 +130,6 @@ entries: wantVersion: "6.17.4+foo", wantPackaged: true, }, - // TODO: Test setting BuildOptions CachedChart and Force. { name: "default values", reference: RemoteReference{Name: "grafana"}, @@ -175,8 +158,10 @@ entries: t.Run(tt.name, func(t *testing.T) { g := NewWithT(t) - targetPath := "/tmp/remote-chart-builder-" + randStringRunes(5) + ".tgz" - defer os.RemoveAll(targetPath) + tmpDir, err := os.MkdirTemp("", "remote-chart-builder-") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpDir) + targetPath := filepath.Join(tmpDir, "chart.tgz") if tt.repository != nil { _, err := tt.repository.CacheIndex() @@ -211,6 +196,73 @@ entries: } } +func TestRemoteBuilder_Build_CachedChart(t *testing.T) { + g := NewWithT(t) + + chartGrafana, err := os.ReadFile("./../testdata/charts/helmchart-0.1.0.tgz") + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(chartGrafana).ToNot(BeEmpty()) + + index := []byte(` +apiVersion: v1 +entries: + grafana: + - urls: + - https://example.com/grafana.tgz + description: string + version: 0.1.0 +`) + + mockGetter := &mockIndexChartGetter{ + IndexResponse: index, + ChartResponse: chartGrafana, + } + mockRepo := func() *repository.ChartRepository { + return &repository.ChartRepository{ + URL: "https://grafana.github.io/helm-charts/", + Client: mockGetter, + RWMutex: &sync.RWMutex{}, + } + } + + reference := RemoteReference{Name: "grafana"} + repository := mockRepo() + + _, err = repository.CacheIndex() + g.Expect(err).ToNot(HaveOccurred()) + // Cleanup the cache index path. + defer os.Remove(repository.CachePath) + + b := NewRemoteBuilder(repository) + + tmpDir, err := os.MkdirTemp("", "remote-chart-") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpDir) + + // Build first time. + targetPath := filepath.Join(tmpDir, "chart1.tgz") + defer os.RemoveAll(targetPath) + buildOpts := BuildOptions{} + cb, err := b.Build(context.TODO(), reference, targetPath, buildOpts) + g.Expect(err).ToNot(HaveOccurred()) + + // Set the result as the CachedChart for second build. + buildOpts.CachedChart = cb.Path + + // Rebuild with a new path. + targetPath2 := filepath.Join(tmpDir, "chart2.tgz") + defer os.RemoveAll(targetPath2) + cb, err = b.Build(context.TODO(), reference, targetPath2, buildOpts) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(cb.Path).To(Equal(targetPath)) + + // Rebuild with build option Force. + buildOpts.Force = true + cb, err = b.Build(context.TODO(), reference, targetPath2, buildOpts) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(cb.Path).To(Equal(targetPath2)) +} + func Test_mergeChartValues(t *testing.T) { tests := []struct { name string From ef0517372b4cf178a809280acfb0577e51f641da Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Tue, 16 Nov 2021 23:32:33 +0100 Subject: [PATCH 13/23] internal/helm: tweak and test chart build summary This makes the string less verbose and deals with the safe handling of some edge-case build states. Signed-off-by: Hidde Beydals --- internal/helm/chart/builder.go | 21 ++--- internal/helm/chart/builder_test.go | 82 ++++++++++++++++++++ internal/helm/repository/chart_repository.go | 2 - 3 files changed, 93 insertions(+), 12 deletions(-) diff --git a/internal/helm/chart/builder.go b/internal/helm/chart/builder.go index 3698d02c1..71bfaf2a7 100644 --- a/internal/helm/chart/builder.go +++ b/internal/helm/chart/builder.go @@ -138,8 +138,8 @@ type Build struct { // Summary returns a human-readable summary of the Build. func (b *Build) Summary() string { - if b == nil { - return "no chart build" + if b == nil || b.Name == "" || b.Version == "" { + return "No chart build." } var s strings.Builder @@ -148,25 +148,26 @@ func (b *Build) Summary() string { if b.Packaged { action = "Packaged" } - s.WriteString(fmt.Sprintf("%s '%s' chart with version '%s'.", action, b.Name, b.Version)) + s.WriteString(fmt.Sprintf("%s '%s' chart with version '%s'", action, b.Name, b.Version)) - if b.Packaged && b.ResolvedDependencies > 0 { - s.WriteString(fmt.Sprintf(" Resolved %d dependencies before packaging.", b.ResolvedDependencies)) + if b.Packaged && len(b.ValueFiles) > 0 { + s.WriteString(fmt.Sprintf(", with merged value files %v", b.ValueFiles)) } - if len(b.ValueFiles) > 0 { - s.WriteString(fmt.Sprintf(" Merged %v value files into default chart values.", b.ValueFiles)) + if b.Packaged && b.ResolvedDependencies > 0 { + s.WriteString(fmt.Sprintf(", resolving %d dependencies before packaging", b.ResolvedDependencies)) } + s.WriteString(".") return s.String() } // String returns the Path of the Build. func (b *Build) String() string { - if b != nil { - return b.Path + if b == nil { + return "" } - return "" + return b.Path } // packageToPath attempts to package the given chart to the out filepath. diff --git a/internal/helm/chart/builder_test.go b/internal/helm/chart/builder_test.go index 92aec74f1..05b3ec1b0 100644 --- a/internal/helm/chart/builder_test.go +++ b/internal/helm/chart/builder_test.go @@ -25,8 +25,90 @@ import ( . "github.com/onsi/gomega" "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/chartutil" ) +func TestBuildOptions_GetValueFiles(t *testing.T) { + tests := []struct { + name string + valueFiles []string + want []string + }{ + { + name: "Default values.yaml", + valueFiles: []string{chartutil.ValuesfileName}, + want: nil, + }, + { + name: "Value files", + valueFiles: []string{chartutil.ValuesfileName, "foo.yaml"}, + want: []string{chartutil.ValuesfileName, "foo.yaml"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + o := BuildOptions{ValueFiles: tt.valueFiles} + g.Expect(o.GetValueFiles()).To(Equal(tt.want)) + }) + } +} + +func TestChartBuildResult_Summary(t *testing.T) { + tests := []struct { + name string + build *Build + want string + }{ + { + name: "Simple", + build: &Build{ + Name: "chart", + Version: "1.2.3-rc.1+bd6bf40", + }, + want: "Fetched 'chart' chart with version '1.2.3-rc.1+bd6bf40'.", + }, + { + name: "With value files", + build: &Build{ + Name: "chart", + Version: "arbitrary-version", + Packaged: true, + ValueFiles: []string{"a.yaml", "b.yaml"}, + }, + want: "Packaged 'chart' chart with version 'arbitrary-version', with merged value files [a.yaml b.yaml].", + }, + { + name: "With dependencies", + build: &Build{ + Name: "chart", + Version: "arbitrary-version", + Packaged: true, + ResolvedDependencies: 5, + }, + want: "Packaged 'chart' chart with version 'arbitrary-version', resolving 5 dependencies before packaging.", + }, + { + name: "Empty build", + build: &Build{}, + want: "No chart build.", + }, + { + name: "Nil build", + build: nil, + want: "No chart build.", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + g.Expect(tt.build.Summary()).To(Equal(tt.want)) + }) + } +} + func TestChartBuildResult_String(t *testing.T) { g := NewWithT(t) diff --git a/internal/helm/repository/chart_repository.go b/internal/helm/repository/chart_repository.go index 638355f80..c9bab590d 100644 --- a/internal/helm/repository/chart_repository.go +++ b/internal/helm/repository/chart_repository.go @@ -287,8 +287,6 @@ func (r *ChartRepository) CacheIndex() (string, error) { // LoadFromCache if it does not HasIndex. // If it not HasCacheFile, a cache attempt is made using CacheIndex // before continuing to load. -// It returns a boolean indicating if it cached the index before -// loading, or an error. func (r *ChartRepository) StrategicallyLoadIndex() (err error) { if r.HasIndex() { return From 4fd6e6ef60d16cd3147679aa607cc49eff35c114 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Wed, 17 Nov 2021 01:31:35 +0100 Subject: [PATCH 14/23] internal/helm: add more tests Signed-off-by: Hidde Beydals --- internal/helm/chart/builder_local_test.go | 3 +- internal/helm/chart/builder_remote_test.go | 25 ++++++ internal/helm/chart/builder_test.go | 76 +++++++++++++++++ internal/helm/chart/dependency_manager.go | 4 +- .../helm/chart/dependency_manager_test.go | 45 +++++++++- internal/helm/getter/mock.go | 41 --------- .../helm/repository/chart_repository_test.go | 83 +++++++++++++++++-- 7 files changed, 222 insertions(+), 55 deletions(-) delete mode 100644 internal/helm/getter/mock.go diff --git a/internal/helm/chart/builder_local_test.go b/internal/helm/chart/builder_local_test.go index 7f42ee905..5691371f2 100644 --- a/internal/helm/chart/builder_local_test.go +++ b/internal/helm/chart/builder_local_test.go @@ -30,7 +30,6 @@ import ( "helm.sh/helm/v3/pkg/chartutil" "helm.sh/helm/v3/pkg/repo" - "github.com/fluxcd/source-controller/internal/helm/getter" "github.com/fluxcd/source-controller/internal/helm/repository" ) @@ -43,7 +42,7 @@ func TestLocalBuilder_Build(t *testing.T) { g.Expect(chartB).ToNot(BeEmpty()) mockRepo := func() *repository.ChartRepository { return &repository.ChartRepository{ - Client: &getter.MockGetter{ + Client: &mockGetter{ Response: chartB, }, Index: &repo.IndexFile{ diff --git a/internal/helm/chart/builder_remote_test.go b/internal/helm/chart/builder_remote_test.go index e8ad6be54..80534c60b 100644 --- a/internal/helm/chart/builder_remote_test.go +++ b/internal/helm/chart/builder_remote_test.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "os" + "path/filepath" "strings" "sync" "testing" @@ -337,6 +338,30 @@ func Test_mergeChartValues(t *testing.T) { } } +func Test_validatePackageAndWriteToPath(t *testing.T) { + g := NewWithT(t) + + tmpDir, err := os.MkdirTemp("", "validate-pkg-chart-") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpDir) + + validF, err := os.Open("./../testdata/charts/helmchart-0.1.0.tgz") + g.Expect(err).ToNot(HaveOccurred()) + defer validF.Close() + + chartPath := filepath.Join(tmpDir, "chart.tgz") + defer os.Remove(chartPath) + err = validatePackageAndWriteToPath(validF, chartPath) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(chartPath).To(BeARegularFile()) + + emptyF, err := os.Open("./../testdata/charts/empty.tgz") + defer emptyF.Close() + g.Expect(err).ToNot(HaveOccurred()) + err = validatePackageAndWriteToPath(emptyF, filepath.Join(tmpDir, "out.tgz")) + g.Expect(err).To(HaveOccurred()) +} + func Test_pathIsDir(t *testing.T) { tests := []struct { name string diff --git a/internal/helm/chart/builder_test.go b/internal/helm/chart/builder_test.go index 05b3ec1b0..87f0b93d2 100644 --- a/internal/helm/chart/builder_test.go +++ b/internal/helm/chart/builder_test.go @@ -28,6 +28,82 @@ import ( "helm.sh/helm/v3/pkg/chartutil" ) +func TestLocalReference_Validate(t *testing.T) { + tests := []struct { + name string + ref LocalReference + wantErr string + }{ + { + name: "ref with path", + ref: LocalReference{Path: "/a/path"}, + }, + { + name: "ref with path and work dir", + ref: LocalReference{Path: "/a/path", WorkDir: "/with/a/workdir"}, + }, + { + name: "ref without path", + ref: LocalReference{WorkDir: "/just/a/workdir"}, + wantErr: "no path set for local chart reference", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + err := tt.ref.Validate() + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + return + } + g.Expect(err).ToNot(HaveOccurred()) + }) + } +} + +func TestRemoteReference_Validate(t *testing.T) { + tests := []struct { + name string + ref RemoteReference + wantErr string + }{ + { + name: "ref with name", + ref: RemoteReference{Name: "valid-chart-name"}, + }, + { + name: "ref with invalid name", + ref: RemoteReference{Name: "iNvAlID-ChArT-NAmE!"}, + wantErr: "invalid chart name 'iNvAlID-ChArT-NAmE!'", + }, + { + name: "ref with Artifactory specific invalid format", + ref: RemoteReference{Name: "i-shall/not"}, + wantErr: "invalid chart name 'i-shall/not'", + }, + { + name: "ref without name", + ref: RemoteReference{}, + wantErr: "no name set for remote chart reference", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + err := tt.ref.Validate() + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + return + } + g.Expect(err).ToNot(HaveOccurred()) + }) + } +} + func TestBuildOptions_GetValueFiles(t *testing.T) { tests := []struct { name string diff --git a/internal/helm/chart/dependency_manager.go b/internal/helm/chart/dependency_manager.go index 2fa1df32c..798f6df92 100644 --- a/internal/helm/chart/dependency_manager.go +++ b/internal/helm/chart/dependency_manager.go @@ -95,7 +95,9 @@ func (dm *DependencyManager) Clear() []error { var errs []error for _, v := range dm.repositories { v.Unload() - errs = append(errs, v.RemoveCache()) + if err := v.RemoveCache(); err != nil { + errs = append(errs, err) + } } return errs } diff --git a/internal/helm/chart/dependency_manager_test.go b/internal/helm/chart/dependency_manager_test.go index da4b70a67..04c0fc46e 100644 --- a/internal/helm/chart/dependency_manager_test.go +++ b/internal/helm/chart/dependency_manager_test.go @@ -17,6 +17,7 @@ limitations under the License. package chart import ( + "bytes" "context" "errors" "fmt" @@ -28,12 +29,48 @@ import ( . "github.com/onsi/gomega" helmchart "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" + helmgetter "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/repo" - "github.com/fluxcd/source-controller/internal/helm/getter" "github.com/fluxcd/source-controller/internal/helm/repository" ) +// mockGetter is a simple mocking getter.Getter implementation, returning +// a byte response to any provided URL. +type mockGetter struct { + Response []byte +} + +func (g *mockGetter) Get(_ string, _ ...helmgetter.Option) (*bytes.Buffer, error) { + r := g.Response + return bytes.NewBuffer(r), nil +} + +func TestDependencyManager_Clear(t *testing.T) { + g := NewWithT(t) + + repos := map[string]*repository.ChartRepository{ + "with index": { + Index: repo.NewIndexFile(), + RWMutex: &sync.RWMutex{}, + }, + "cached cache path": { + CachePath: "/invalid/path/resets", + Cached: true, + RWMutex: &sync.RWMutex{}, + }, + } + + dm := NewDependencyManager(WithRepositories(repos)) + g.Expect(dm.Clear()).To(BeNil()) + g.Expect(dm.repositories).To(HaveLen(len(repos))) + for _, v := range repos { + g.Expect(v.Index).To(BeNil()) + g.Expect(v.CachePath).To(BeEmpty()) + g.Expect(v.Cached).To(BeFalse()) + } +} + func TestDependencyManager_Build(t *testing.T) { g := NewWithT(t) @@ -45,7 +82,7 @@ func TestDependencyManager_Build(t *testing.T) { mockRepo := func() *repository.ChartRepository { return &repository.ChartRepository{ - Client: &getter.MockGetter{ + Client: &mockGetter{ Response: chartGrafana, }, Index: &repo.IndexFile{ @@ -286,7 +323,7 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) { name: "adds remote dependency", repositories: map[string]*repository.ChartRepository{ "https://example.com/": { - Client: &getter.MockGetter{ + Client: &mockGetter{ Response: chartB, }, Index: &repo.IndexFile{ @@ -403,7 +440,7 @@ func TestDependencyManager_addRemoteDependency(t *testing.T) { name: "chart load error", repositories: map[string]*repository.ChartRepository{ "https://example.com/": { - Client: &getter.MockGetter{}, + Client: &mockGetter{}, Index: &repo.IndexFile{ Entries: map[string]repo.ChartVersions{ chartName: { diff --git a/internal/helm/getter/mock.go b/internal/helm/getter/mock.go deleted file mode 100644 index 91cd2b7bc..000000000 --- a/internal/helm/getter/mock.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2021 The Flux authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package getter - -import ( - "bytes" - - "helm.sh/helm/v3/pkg/getter" -) - -// MockGetter can be used as a simple mocking getter.Getter implementation. -type MockGetter struct { - Response []byte - - requestedURL string -} - -func (g *MockGetter) Get(u string, _ ...getter.Option) (*bytes.Buffer, error) { - g.requestedURL = u - r := g.Response - return bytes.NewBuffer(r), nil -} - -// LastGet returns the last requested URL for Get. -func (g *MockGetter) LastGet() string { - return g.requestedURL -} diff --git a/internal/helm/repository/chart_repository_test.go b/internal/helm/repository/chart_repository_test.go index b6f191f3b..22d3e664b 100644 --- a/internal/helm/repository/chart_repository_test.go +++ b/internal/helm/repository/chart_repository_test.go @@ -29,8 +29,6 @@ import ( "helm.sh/helm/v3/pkg/chart" helmgetter "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/repo" - - "github.com/fluxcd/source-controller/internal/helm/getter" ) var now = time.Now() @@ -41,6 +39,19 @@ const ( unorderedTestFile = "../testdata/local-index-unordered.yaml" ) +// mockGetter is a simple mocking getter.Getter implementation, returning +// a byte response to any provided URL. +type mockGetter struct { + Response []byte + LastCalledURL string +} + +func (g *mockGetter) Get(u string, _ ...helmgetter.Option) (*bytes.Buffer, error) { + r := g.Response + g.LastCalledURL = u + return bytes.NewBuffer(r), nil +} + func TestNewChartRepository(t *testing.T) { repositoryURL := "https://example.com" providers := helmgetter.Providers{ @@ -220,7 +231,7 @@ func TestChartRepository_DownloadChart(t *testing.T) { g := NewWithT(t) t.Parallel() - mg := getter.MockGetter{} + mg := mockGetter{} r := &ChartRepository{ URL: tt.url, Client: &mg, @@ -231,7 +242,7 @@ func TestChartRepository_DownloadChart(t *testing.T) { g.Expect(res).To(BeNil()) return } - g.Expect(mg.LastGet()).To(Equal(tt.wantURL)) + g.Expect(mg.LastCalledURL).To(Equal(tt.wantURL)) g.Expect(res).ToNot(BeNil()) g.Expect(err).ToNot(HaveOccurred()) }) @@ -244,7 +255,7 @@ func TestChartRepository_DownloadIndex(t *testing.T) { b, err := os.ReadFile(chartmuseumTestFile) g.Expect(err).ToNot(HaveOccurred()) - mg := getter.MockGetter{Response: b} + mg := mockGetter{Response: b} r := &ChartRepository{ URL: "https://example.com", Client: &mg, @@ -253,7 +264,7 @@ func TestChartRepository_DownloadIndex(t *testing.T) { buf := bytes.NewBuffer([]byte{}) g.Expect(r.DownloadIndex(buf)).To(Succeed()) g.Expect(buf.Bytes()).To(Equal(b)) - g.Expect(mg.LastGet()).To(Equal(r.URL + "/index.yaml")) + g.Expect(mg.LastCalledURL).To(Equal(r.URL + "/index.yaml")) g.Expect(err).To(BeNil()) } @@ -374,7 +385,7 @@ func TestChartRepository_LoadIndexFromFile(t *testing.T) { func TestChartRepository_CacheIndex(t *testing.T) { g := NewWithT(t) - mg := getter.MockGetter{Response: []byte("foo")} + mg := mockGetter{Response: []byte("foo")} expectSum := fmt.Sprintf("%x", sha256.Sum256(mg.Response)) r := newChartRepository() @@ -393,6 +404,31 @@ func TestChartRepository_CacheIndex(t *testing.T) { g.Expect(sum).To(BeEquivalentTo(expectSum)) } +func TestChartRepository_StrategicallyLoadIndex(t *testing.T) { + g := NewWithT(t) + + r := newChartRepository() + r.Index = repo.NewIndexFile() + g.Expect(r.StrategicallyLoadIndex()).To(Succeed()) + g.Expect(r.CachePath).To(BeEmpty()) + g.Expect(r.Cached).To(BeFalse()) + + r.Index = nil + r.CachePath = "/invalid/cache/index/path.yaml" + err := r.StrategicallyLoadIndex() + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("/invalid/cache/index/path.yaml: no such file or directory")) + g.Expect(r.Cached).To(BeFalse()) + + r.CachePath = "" + r.Client = &mockGetter{} + err = r.StrategicallyLoadIndex() + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("no API version specified")) + g.Expect(r.Cached).To(BeTrue()) + g.Expect(r.RemoveCache()).To(Succeed()) +} + func TestChartRepository_LoadFromCache(t *testing.T) { tests := []struct { name string @@ -443,6 +479,15 @@ func TestChartRepository_HasIndex(t *testing.T) { g.Expect(r.HasIndex()).To(BeTrue()) } +func TestChartRepository_HasCacheFile(t *testing.T) { + g := NewWithT(t) + + r := newChartRepository() + g.Expect(r.HasCacheFile()).To(BeFalse()) + r.CachePath = "foo" + g.Expect(r.HasCacheFile()).To(BeTrue()) +} + func TestChartRepository_UnloadIndex(t *testing.T) { g := NewWithT(t) @@ -522,3 +567,27 @@ func verifyLocalIndex(t *testing.T, i *repo.IndexFile) { g.Expect(tt.Keywords).To(ContainElements(expect.Keywords)) } } + +func TestChartRepository_RemoveCache(t *testing.T) { + g := NewWithT(t) + + tmpFile, err := os.CreateTemp("", "remove-cache-") + g.Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tmpFile.Name()) + + r := newChartRepository() + r.CachePath = tmpFile.Name() + r.Cached = true + + g.Expect(r.RemoveCache()).To(Succeed()) + g.Expect(r.CachePath).To(BeEmpty()) + g.Expect(r.Cached).To(BeFalse()) + g.Expect(tmpFile.Name()).ToNot(BeAnExistingFile()) + + r.CachePath = tmpFile.Name() + r.Cached = true + + g.Expect(r.RemoveCache()).To(Succeed()) + g.Expect(r.CachePath).To(BeEmpty()) + g.Expect(r.Cached).To(BeFalse()) +} From 2b8134ce20374d1d820962183e21bf8481ea93ab Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Wed, 17 Nov 2021 22:46:23 +0100 Subject: [PATCH 15/23] internal/helm: introduce typed BuildError This commit introduces a typed `BuildError` to be returned by `Builder.Build` in case of a failure. The `Reason` field in combination with `BuildErrorReason` can be used to signal (or determine) the reason of a returned error within the context of the build process. At present this is used to determine the correct Condition Reason, but in a future iteration this can be used to determine the negative polarity condition that should be set to indicate a precise failure to the user. Signed-off-by: Hidde Beydals --- controllers/helmchart_controller.go | 43 ++++++++++---- internal/helm/chart/builder.go | 2 +- internal/helm/chart/builder_local.go | 27 +++++---- internal/helm/chart/builder_remote.go | 32 +++++----- internal/helm/chart/builder_test.go | 2 +- internal/helm/chart/errors.go | 65 +++++++++++++++++++++ internal/helm/chart/errors_test.go | 84 +++++++++++++++++++++++++++ 7 files changed, 217 insertions(+), 38 deletions(-) create mode 100644 internal/helm/chart/errors.go create mode 100644 internal/helm/chart/errors_test.go diff --git a/controllers/helmchart_controller.go b/controllers/helmchart_controller.go index 3c1be0a7d..0e0b2cd23 100644 --- a/controllers/helmchart_controller.go +++ b/controllers/helmchart_controller.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "errors" "fmt" "net/url" "os" @@ -29,7 +30,7 @@ import ( "github.com/go-logr/logr" helmgetter "helm.sh/helm/v3/pkg/getter" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" + apierrs "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -445,19 +446,19 @@ func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source so } // Build chart - chartB := chart.NewLocalBuilder(dm) - build, err := chartB.Build(ctx, chart.LocalReference{WorkDir: sourceDir, Path: chartPath}, filepath.Join(workDir, "chart.tgz"), buildsOpts) + chartBuilder := chart.NewLocalBuilder(dm) + result, err := chartBuilder.Build(ctx, chart.LocalReference{WorkDir: sourceDir, Path: chartPath}, filepath.Join(workDir, "chart.tgz"), buildsOpts) if err != nil { - return sourcev1.HelmChartNotReady(c, sourcev1.ChartPackageFailedReason, err.Error()), err + return sourcev1.HelmChartNotReady(c, reasonForBuildError(err), err.Error()), err } - newArtifact := r.Storage.NewArtifactFor(c.Kind, c.GetObjectMeta(), build.Version, - fmt.Sprintf("%s-%s.tgz", build.Name, build.Version)) + newArtifact := r.Storage.NewArtifactFor(c.Kind, c.GetObjectMeta(), result.Version, + fmt.Sprintf("%s-%s.tgz", result.Name, result.Version)) // If the path of the returned build equals the cache path, // there are no changes to the chart if apimeta.IsStatusConditionTrue(c.Status.Conditions, meta.ReadyCondition) && - build.Path == buildsOpts.CachedChart { + result.Path == buildsOpts.CachedChart { // Ensure hostname is updated if c.GetArtifact().URL != newArtifact.URL { r.Storage.SetArtifactURL(c.GetArtifact()) @@ -482,19 +483,19 @@ func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source so defer unlock() // Copy the packaged chart to the artifact path - if err = r.Storage.CopyFromPath(&newArtifact, build.Path); err != nil { + if err = r.Storage.CopyFromPath(&newArtifact, result.Path); err != nil { err = fmt.Errorf("failed to write chart package to storage: %w", err) return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } // Update symlink - cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", build.Name)) + cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", result.Name)) if err != nil { err = fmt.Errorf("storage error: %w", err) return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } - return sourcev1.HelmChartReady(c, newArtifact, cUrl, sourcev1.ChartPackageSucceededReason, build.Summary()), nil + return sourcev1.HelmChartReady(c, newArtifact, cUrl, reasonForBuildSuccess(result), result.Summary()), nil } // namespacedChartRepositoryCallback returns a chart.GetChartRepositoryCallback @@ -508,7 +509,7 @@ func (r *HelmChartReconciler) namespacedChartRepositoryCallback(ctx context.Cont repo, err := r.resolveDependencyRepository(ctx, url, namespace) if err != nil { // Return Kubernetes client errors, but ignore others - if errors.ReasonForError(err) != metav1.StatusReasonUnknown { + if apierrs.ReasonForError(err) != metav1.StatusReasonUnknown { return nil, err } repo = &sourcev1.HelmRepository{ @@ -807,3 +808,23 @@ func (r *HelmChartReconciler) recordSuspension(ctx context.Context, chart source r.MetricsRecorder.RecordSuspend(*objRef, chart.Spec.Suspend) } } + +func reasonForBuildError(err error) string { + var buildErr *chart.BuildError + if ok := errors.As(err, &buildErr); !ok { + return sourcev1.ChartPullFailedReason + } + switch buildErr.Reason { + case chart.ErrChartMetadataPatch, chart.ErrValueFilesMerge, chart.ErrDependencyBuild, chart.ErrChartPackage: + return sourcev1.ChartPackageFailedReason + default: + return sourcev1.ChartPullFailedReason + } +} + +func reasonForBuildSuccess(result *chart.Build) string { + if result.Packaged { + return sourcev1.ChartPackageSucceededReason + } + return sourcev1.ChartPullSucceededReason +} diff --git a/internal/helm/chart/builder.go b/internal/helm/chart/builder.go index 71bfaf2a7..d1b0f747c 100644 --- a/internal/helm/chart/builder.go +++ b/internal/helm/chart/builder.go @@ -144,7 +144,7 @@ func (b *Build) Summary() string { var s strings.Builder - action := "Fetched" + var action = "Pulled" if b.Packaged { action = "Packaged" } diff --git a/internal/helm/chart/builder_local.go b/internal/helm/chart/builder_local.go index 037a2fe18..2f27b8b28 100644 --- a/internal/helm/chart/builder_local.go +++ b/internal/helm/chart/builder_local.go @@ -51,14 +51,14 @@ func (b *localChartBuilder) Build(ctx context.Context, ref Reference, p string, } if err := ref.Validate(); err != nil { - return nil, err + return nil, &BuildError{Reason: ErrChartPull, Err: err} } // Load the chart metadata from the LocalReference to ensure it points // to a chart curMeta, err := LoadChartMetadata(localRef.Path) if err != nil { - return nil, err + return nil, &BuildError{Reason: ErrChartPull, Err: err} } result := &Build{} @@ -69,10 +69,12 @@ func (b *localChartBuilder) Build(ctx context.Context, ref Reference, p string, if opts.VersionMetadata != "" { ver, err := semver.NewVersion(curMeta.Version) if err != nil { - return nil, fmt.Errorf("failed to parse chart version from metadata as SemVer: %w", err) + err = fmt.Errorf("failed to parse version from chart metadata as SemVer: %w", err) + return nil, &BuildError{Reason: ErrChartMetadataPatch, Err: err} } if *ver, err = ver.SetMetadata(opts.VersionMetadata); err != nil { - return nil, fmt.Errorf("failed to set metadata on chart version: %w", err) + err = fmt.Errorf("failed to set SemVer metadata on chart version: %w", err) + return nil, &BuildError{Reason: ErrChartMetadataPatch, Err: err} } result.Version = ver.String() } @@ -92,8 +94,8 @@ func (b *localChartBuilder) Build(ctx context.Context, ref Reference, p string, // options are set, we can copy the chart without making modifications isChartDir := pathIsDir(localRef.Path) if !isChartDir && len(opts.GetValueFiles()) == 0 { - if err := copyFileToPath(localRef.Path, p); err != nil { - return nil, err + if err = copyFileToPath(localRef.Path, p); err != nil { + return nil, &BuildError{Reason: ErrChartPull, Err: err} } result.Path = p return result, nil @@ -103,7 +105,7 @@ func (b *localChartBuilder) Build(ctx context.Context, ref Reference, p string, var mergedValues map[string]interface{} if len(opts.GetValueFiles()) > 0 { if mergedValues, err = mergeFileValues(localRef.WorkDir, opts.ValueFiles); err != nil { - return nil, fmt.Errorf("failed to merge value files: %w", err) + return nil, &BuildError{Reason: ErrValueFilesMerge, Err: err} } } @@ -112,7 +114,7 @@ func (b *localChartBuilder) Build(ctx context.Context, ref Reference, p string, // or because we have merged values and need to repackage chart, err := loader.Load(localRef.Path) if err != nil { - return nil, err + return nil, &BuildError{Reason: ErrChartPackage, Err: err} } // Set earlier resolved version (with metadata) chart.Metadata.Version = result.Version @@ -120,7 +122,7 @@ func (b *localChartBuilder) Build(ctx context.Context, ref Reference, p string, // Overwrite default values with merged values, if any if ok, err = OverwriteChartDefaultValues(chart, mergedValues); ok || err != nil { if err != nil { - return nil, err + return nil, &BuildError{Reason: ErrValueFilesMerge, Err: err} } result.ValueFiles = opts.GetValueFiles() } @@ -128,16 +130,17 @@ func (b *localChartBuilder) Build(ctx context.Context, ref Reference, p string, // Ensure dependencies are fetched if building from a directory if isChartDir { if b.dm == nil { - return nil, fmt.Errorf("local chart builder requires dependency manager for unpackaged charts") + err = fmt.Errorf("local chart builder requires dependency manager for unpackaged charts") + return nil, &BuildError{Reason: ErrDependencyBuild, Err: err} } if result.ResolvedDependencies, err = b.dm.Build(ctx, ref, chart); err != nil { - return nil, err + return nil, &BuildError{Reason: ErrDependencyBuild, Err: err} } } // Package the chart if err = packageToPath(chart, p); err != nil { - return nil, err + return nil, &BuildError{Reason: ErrChartPackage, Err: err} } result.Path = p result.Packaged = true diff --git a/internal/helm/chart/builder_remote.go b/internal/helm/chart/builder_remote.go index 2caceb39c..f03c1a8d2 100644 --- a/internal/helm/chart/builder_remote.go +++ b/internal/helm/chart/builder_remote.go @@ -54,18 +54,20 @@ func (b *remoteChartBuilder) Build(_ context.Context, ref Reference, p string, o } if err := ref.Validate(); err != nil { - return nil, err + return nil, &BuildError{Reason: ErrChartPull, Err: err} } if err := b.remote.LoadFromCache(); err != nil { - return nil, fmt.Errorf("could not load repository index for remote chart reference: %w", err) + err = fmt.Errorf("could not load repository index for remote chart reference: %w", err) + return nil, &BuildError{Reason: ErrChartPull, Err: err} } defer b.remote.Unload() // Get the current version for the RemoteReference cv, err := b.remote.Get(remoteRef.Name, remoteRef.Version) if err != nil { - return nil, fmt.Errorf("failed to get chart version for remote reference: %w", err) + err = fmt.Errorf("failed to get chart version for remote reference: %w", err) + return nil, &BuildError{Reason: ErrChartPull, Err: err} } result := &Build{} @@ -75,10 +77,12 @@ func (b *remoteChartBuilder) Build(_ context.Context, ref Reference, p string, o if opts.VersionMetadata != "" { ver, err := semver.NewVersion(result.Version) if err != nil { - return nil, err + err = fmt.Errorf("failed to parse version from chart metadata as SemVer: %w", err) + return nil, &BuildError{Reason: ErrChartMetadataPatch, Err: err} } if *ver, err = ver.SetMetadata(opts.VersionMetadata); err != nil { - return nil, err + err = fmt.Errorf("failed to set SemVer metadata on chart version: %w", err) + return nil, &BuildError{Reason: ErrChartMetadataPatch, Err: err} } result.Version = ver.String() } @@ -97,14 +101,15 @@ func (b *remoteChartBuilder) Build(_ context.Context, ref Reference, p string, o // Download the package for the resolved version res, err := b.remote.DownloadChart(cv) if err != nil { - return nil, fmt.Errorf("failed to download chart for remote reference: %w", err) + err = fmt.Errorf("failed to download chart for remote reference: %w", err) + return nil, &BuildError{Reason: ErrChartPull, Err: err} } // Use literal chart copy from remote if no custom value files options are // set or build option version metadata isn't set. if len(opts.GetValueFiles()) == 0 && opts.VersionMetadata == "" { if err = validatePackageAndWriteToPath(res, p); err != nil { - return nil, err + return nil, &BuildError{Reason: ErrChartPull, Err: err} } result.Path = p return result, nil @@ -113,26 +118,27 @@ func (b *remoteChartBuilder) Build(_ context.Context, ref Reference, p string, o // Load the chart and merge chart values var chart *helmchart.Chart if chart, err = loader.LoadArchive(res); err != nil { - return nil, fmt.Errorf("failed to load downloaded chart: %w", err) + err = fmt.Errorf("failed to load downloaded chart: %w", err) + return nil, &BuildError{Reason: ErrChartPackage, Err: err} } + chart.Metadata.Version = result.Version mergedValues, err := mergeChartValues(chart, opts.ValueFiles) if err != nil { - return nil, fmt.Errorf("failed to merge chart values: %w", err) + err = fmt.Errorf("failed to merge chart values: %w", err) + return nil, &BuildError{Reason: ErrValueFilesMerge, Err: err} } // Overwrite default values with merged values, if any if ok, err = OverwriteChartDefaultValues(chart, mergedValues); ok || err != nil { if err != nil { - return nil, err + return nil, &BuildError{Reason: ErrValueFilesMerge, Err: err} } result.ValueFiles = opts.GetValueFiles() } - chart.Metadata.Version = result.Version - // Package the chart with the custom values if err = packageToPath(chart, p); err != nil { - return nil, err + return nil, &BuildError{Reason: ErrChartPackage, Err: err} } result.Path = p result.Packaged = true diff --git a/internal/helm/chart/builder_test.go b/internal/helm/chart/builder_test.go index 87f0b93d2..cd64ac41f 100644 --- a/internal/helm/chart/builder_test.go +++ b/internal/helm/chart/builder_test.go @@ -143,7 +143,7 @@ func TestChartBuildResult_Summary(t *testing.T) { Name: "chart", Version: "1.2.3-rc.1+bd6bf40", }, - want: "Fetched 'chart' chart with version '1.2.3-rc.1+bd6bf40'.", + want: "Pulled 'chart' chart with version '1.2.3-rc.1+bd6bf40'.", }, { name: "With value files", diff --git a/internal/helm/chart/errors.go b/internal/helm/chart/errors.go new file mode 100644 index 000000000..746017f23 --- /dev/null +++ b/internal/helm/chart/errors.go @@ -0,0 +1,65 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package chart + +import ( + "errors" + "fmt" +) + +// BuildErrorReason is the descriptive reason for a BuildError. +type BuildErrorReason string + +// Error returns the string representation of BuildErrorReason. +func (e BuildErrorReason) Error() string { + return string(e) +} + +// BuildError contains a wrapped Err and a Reason indicating why it occurred. +type BuildError struct { + Reason error + Err error +} + +// Error returns Err as a string, prefixed with the Reason, if any. +func (e *BuildError) Error() string { + if e.Reason == nil { + return e.Err.Error() + } + return fmt.Sprintf("%s: %s", e.Reason.Error(), e.Err.Error()) +} + +// Is returns true of the Reason or Err equals target. +func (e *BuildError) Is(target error) bool { + if e.Reason != nil && e.Reason == target { + return true + } + return errors.Is(e.Err, target) +} + +// Unwrap returns the underlying Err. +func (e *BuildError) Unwrap() error { + return e.Err +} + +var ( + ErrChartPull = BuildErrorReason("chart pull error") + ErrChartMetadataPatch = BuildErrorReason("chart metadata patch error") + ErrValueFilesMerge = BuildErrorReason("value files merge error") + ErrDependencyBuild = BuildErrorReason("dependency build error") + ErrChartPackage = BuildErrorReason("chart package error") +) diff --git a/internal/helm/chart/errors_test.go b/internal/helm/chart/errors_test.go new file mode 100644 index 000000000..7a33c5431 --- /dev/null +++ b/internal/helm/chart/errors_test.go @@ -0,0 +1,84 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package chart + +import ( + "errors" + "testing" + + . "github.com/onsi/gomega" +) + +func TestBuildErrorReason_Error(t *testing.T) { + g := NewWithT(t) + + err := BuildErrorReason("reason") + g.Expect(err.Error()).To(Equal("reason")) +} + +func TestBuildError_Error(t *testing.T) { + tests := []struct { + name string + err *BuildError + want string + }{ + { + name: "with reason", + err: &BuildError{ + Reason: BuildErrorReason("reason"), + Err: errors.New("error"), + }, + want: "reason: error", + }, + { + name: "without reason", + err: &BuildError{ + Err: errors.New("error"), + }, + want: "error", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + g.Expect(tt.err.Error()).To(Equal(tt.want)) + }) + } +} + +func TestBuildError_Is(t *testing.T) { + g := NewWithT(t) + + wrappedErr := errors.New("wrapped") + err := &BuildError{ + Reason: ErrChartPackage, + Err: wrappedErr, + } + + g.Expect(err.Is(ErrChartPackage)).To(BeTrue()) + g.Expect(err.Is(wrappedErr)).To(BeTrue()) + g.Expect(err.Is(ErrDependencyBuild)).To(BeFalse()) +} + +func TestBuildError_Unwrap(t *testing.T) { + g := NewWithT(t) + + wrap := errors.New("wrapped") + err := BuildError{Err: wrap} + g.Expect(err.Unwrap()).To(Equal(wrap)) +} From 37ac5a9679d9c6e03e359071c524b0c3c42c3e74 Mon Sep 17 00:00:00 2001 From: Sunny Date: Thu, 18 Nov 2021 03:58:54 +0530 Subject: [PATCH 16/23] internal/helm: test load funcs for max size cases This includes a change of the defaults to more acceptible (higher) values. Signed-off-by: Sunny --- internal/helm/chart/metadata_test.go | 36 +++++++++++++++++++ internal/helm/helm.go | 4 +-- .../helm/repository/chart_repository_test.go | 27 ++++++++++++-- 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/internal/helm/chart/metadata_test.go b/internal/helm/chart/metadata_test.go index f2294ff6b..d9c882f43 100644 --- a/internal/helm/chart/metadata_test.go +++ b/internal/helm/chart/metadata_test.go @@ -17,11 +17,16 @@ limitations under the License. package chart import ( + "os" + "path/filepath" "testing" . "github.com/onsi/gomega" + "github.com/otiai10/copy" helmchart "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chartutil" + + "github.com/fluxcd/source-controller/internal/helm" ) var ( @@ -126,6 +131,17 @@ func TestOverwriteChartDefaultValues(t *testing.T) { } func TestLoadChartMetadataFromDir(t *testing.T) { + g := NewWithT(t) + + // Create a chart file that exceeds the max chart file size. + tmpDir, err := os.MkdirTemp("", "load-chart-") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpDir) + copy.Copy("../testdata/charts/helmchart", tmpDir) + bigRequirementsFile := filepath.Join(tmpDir, "requirements.yaml") + data := make([]byte, helm.MaxChartFileSize+10) + g.Expect(os.WriteFile(bigRequirementsFile, data, 0644)).ToNot(HaveOccurred()) + tests := []struct { name string dir string @@ -152,6 +168,11 @@ func TestLoadChartMetadataFromDir(t *testing.T) { dir: "../testdata/charts/", wantErr: "../testdata/charts/Chart.yaml: no such file or directory", }, + { + name: "Error if file size exceeds max size", + dir: tmpDir, + wantErr: "size of 'requirements.yaml' exceeds", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -176,6 +197,16 @@ func TestLoadChartMetadataFromDir(t *testing.T) { } func TestLoadChartMetadataFromArchive(t *testing.T) { + g := NewWithT(t) + + // Create a chart archive that exceeds the max chart size. + tmpDir, err := os.MkdirTemp("", "load-chart-") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpDir) + bigArchiveFile := filepath.Join(tmpDir, "chart.tgz") + data := make([]byte, helm.MaxChartSize+10) + g.Expect(os.WriteFile(bigArchiveFile, data, 0644)).ToNot(HaveOccurred()) + tests := []struct { name string archive string @@ -207,6 +238,11 @@ func TestLoadChartMetadataFromArchive(t *testing.T) { archive: "../testdata/charts/empty.tgz", wantErr: "no 'Chart.yaml' found", }, + { + name: "Error if archive size exceeds max size", + archive: bigArchiveFile, + wantErr: "size of chart 'chart.tgz' exceeds", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/helm/helm.go b/internal/helm/helm.go index ec9668542..854a1ab7b 100644 --- a/internal/helm/helm.go +++ b/internal/helm/helm.go @@ -22,8 +22,8 @@ var ( // MaxIndexSize is the max allowed file size in bytes of a ChartRepository. MaxIndexSize int64 = 50 << 20 // MaxChartSize is the max allowed file size in bytes of a Helm Chart. - MaxChartSize int64 = 2 << 20 + MaxChartSize int64 = 10 << 20 // MaxChartFileSize is the max allowed file size in bytes of any arbitrary // file originating from a chart. - MaxChartFileSize int64 = 2 << 10 + MaxChartFileSize int64 = 5 << 20 ) diff --git a/internal/helm/repository/chart_repository_test.go b/internal/helm/repository/chart_repository_test.go index 22d3e664b..c0100dd3d 100644 --- a/internal/helm/repository/chart_repository_test.go +++ b/internal/helm/repository/chart_repository_test.go @@ -22,9 +22,11 @@ import ( "fmt" "net/url" "os" + "path/filepath" "testing" "time" + "github.com/fluxcd/source-controller/internal/helm" . "github.com/onsi/gomega" "helm.sh/helm/v3/pkg/chart" helmgetter "helm.sh/helm/v3/pkg/getter" @@ -353,9 +355,20 @@ func TestChartRepository_LoadIndexFromBytes_Unordered(t *testing.T) { // Index load tests are derived from https://github.com/helm/helm/blob/v3.3.4/pkg/repo/index_test.go#L108 // to ensure parity with Helm behaviour. func TestChartRepository_LoadIndexFromFile(t *testing.T) { + g := NewWithT(t) + + // Create an index file that exceeds the max index size. + tmpDir, err := os.MkdirTemp("", "load-index-") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpDir) + bigIndexFile := filepath.Join(tmpDir, "index.yaml") + data := make([]byte, helm.MaxIndexSize+10) + g.Expect(os.WriteFile(bigIndexFile, data, 0644)).ToNot(HaveOccurred()) + tests := []struct { name string filename string + wantErr string }{ { name: "regular index file", @@ -365,16 +378,26 @@ func TestChartRepository_LoadIndexFromFile(t *testing.T) { name: "chartmuseum index file", filename: chartmuseumTestFile, }, + { + name: "error if index size exceeds max size", + filename: bigIndexFile, + wantErr: "size of index 'index.yaml' exceeds", + }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { g := NewWithT(t) - t.Parallel() r := newChartRepository() - err := r.LoadFromFile(testFile) + err := r.LoadFromFile(tt.filename) + if tt.wantErr != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErr)) + return + } + g.Expect(err).ToNot(HaveOccurred()) verifyLocalIndex(t, r.Index) From a1e9302b7dce0ce73496de4379394e63f1002fe6 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 18 Nov 2021 09:24:34 +0100 Subject: [PATCH 17/23] internal/helm: "value files" -> "values files" Previous usage while consistent, was incorrect, and inconsitent with the field in the API spec. Signed-off-by: Hidde Beydals --- controllers/helmchart_controller.go | 8 ++--- internal/helm/chart/builder.go | 32 +++++++++---------- internal/helm/chart/builder_local.go | 16 +++++----- internal/helm/chart/builder_local_test.go | 10 +++--- internal/helm/chart/builder_remote.go | 14 ++++----- internal/helm/chart/builder_remote_test.go | 2 +- internal/helm/chart/builder_test.go | 36 +++++++++++----------- internal/helm/chart/dependency_manager.go | 2 +- internal/helm/chart/errors.go | 2 +- internal/helm/chart/errors_test.go | 10 +++--- 10 files changed, 66 insertions(+), 66 deletions(-) diff --git a/controllers/helmchart_controller.go b/controllers/helmchart_controller.go index 0e0b2cd23..685a43a57 100644 --- a/controllers/helmchart_controller.go +++ b/controllers/helmchart_controller.go @@ -335,7 +335,7 @@ func (r *HelmChartReconciler) fromHelmRepository(ctx context.Context, repo sourc cBuilder := chart.NewRemoteBuilder(chartRepo) ref := chart.RemoteReference{Name: c.Spec.Chart, Version: c.Spec.Version} opts := chart.BuildOptions{ - ValueFiles: c.GetValuesFiles(), + ValuesFiles: c.GetValuesFiles(), CachedChart: cachedChart, Force: force, } @@ -431,8 +431,8 @@ func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source so // Configure builder options, including any previously cached chart buildsOpts := chart.BuildOptions{ - ValueFiles: c.GetValuesFiles(), - Force: force, + ValuesFiles: c.GetValuesFiles(), + Force: force, } if artifact := c.Status.Artifact; artifact != nil { buildsOpts.CachedChart = artifact.Path @@ -815,7 +815,7 @@ func reasonForBuildError(err error) string { return sourcev1.ChartPullFailedReason } switch buildErr.Reason { - case chart.ErrChartMetadataPatch, chart.ErrValueFilesMerge, chart.ErrDependencyBuild, chart.ErrChartPackage: + case chart.ErrChartMetadataPatch, chart.ErrValuesFilesMerge, chart.ErrDependencyBuild, chart.ErrChartPackage: return sourcev1.ChartPackageFailedReason default: return sourcev1.ChartPullFailedReason diff --git a/internal/helm/chart/builder.go b/internal/helm/chart/builder.go index d1b0f747c..9aa2a17e4 100644 --- a/internal/helm/chart/builder.go +++ b/internal/helm/chart/builder.go @@ -81,10 +81,10 @@ func (r RemoteReference) Validate() error { // Builder is capable of building a (specific) chart Reference. type Builder interface { - // Build builds and packages a Helm chart with the given Reference - // and BuildOptions and writes it to p. It returns the Build result, - // or an error. It may return an error for unsupported Reference - // implementations. + // Build pulls and (optionally) packages a Helm chart with the given + // Reference and BuildOptions, and writes it to p. + // It returns the Build result, or an error. + // It may return an error for unsupported Reference implementations. Build(ctx context.Context, ref Reference, p string, opts BuildOptions) (*Build, error) } @@ -94,25 +94,25 @@ type BuildOptions struct { // the spec, and is included during packaging. // Ref: https://semver.org/#spec-item-10 VersionMetadata string - // ValueFiles can be set to a list of relative paths, used to compose + // ValuesFiles can be set to a list of relative paths, used to compose // and overwrite an alternative default "values.yaml" for the chart. - ValueFiles []string + ValuesFiles []string // CachedChart can be set to the absolute path of a chart stored on // the local filesystem, and is used for simple validation by metadata // comparisons. CachedChart string // Force can be set to force the build of the chart, for example - // because the list of ValueFiles has changed. + // because the list of ValuesFiles has changed. Force bool } -// GetValueFiles returns BuildOptions.ValueFiles, except if it equals +// GetValuesFiles returns BuildOptions.ValuesFiles, except if it equals // "values.yaml", which returns nil. -func (o BuildOptions) GetValueFiles() []string { - if len(o.ValueFiles) == 1 && filepath.Clean(o.ValueFiles[0]) == filepath.Clean(chartutil.ValuesfileName) { +func (o BuildOptions) GetValuesFiles() []string { + if len(o.ValuesFiles) == 1 && filepath.Clean(o.ValuesFiles[0]) == filepath.Clean(chartutil.ValuesfileName) { return nil } - return o.ValueFiles + return o.ValuesFiles } // Build contains the Builder.Build result, including specific @@ -124,14 +124,14 @@ type Build struct { Name string // Version of the packaged chart. Version string - // ValueFiles is the list of files used to compose the chart's + // ValuesFiles is the list of files used to compose the chart's // default "values.yaml". - ValueFiles []string + ValuesFiles []string // ResolvedDependencies is the number of local and remote dependencies // collected by the DependencyManager before building the chart. ResolvedDependencies int // Packaged indicates if the Builder has packaged the chart. - // This can for example be false if ValueFiles is empty and the chart + // This can for example be false if ValuesFiles is empty and the chart // source was already packaged. Packaged bool } @@ -150,8 +150,8 @@ func (b *Build) Summary() string { } s.WriteString(fmt.Sprintf("%s '%s' chart with version '%s'", action, b.Name, b.Version)) - if b.Packaged && len(b.ValueFiles) > 0 { - s.WriteString(fmt.Sprintf(", with merged value files %v", b.ValueFiles)) + if b.Packaged && len(b.ValuesFiles) > 0 { + s.WriteString(fmt.Sprintf(", with merged values files %v", b.ValuesFiles)) } if b.Packaged && b.ResolvedDependencies > 0 { diff --git a/internal/helm/chart/builder_local.go b/internal/helm/chart/builder_local.go index 2f27b8b28..5d79da3df 100644 --- a/internal/helm/chart/builder_local.go +++ b/internal/helm/chart/builder_local.go @@ -85,15 +85,15 @@ func (b *localChartBuilder) Build(ctx context.Context, ref Reference, p string, if opts.CachedChart != "" && !opts.Force { if curMeta, err = LoadChartMetadataFromArchive(opts.CachedChart); err == nil && result.Version == curMeta.Version { result.Path = opts.CachedChart - result.ValueFiles = opts.ValueFiles + result.ValuesFiles = opts.ValuesFiles return result, nil } } - // If the chart at the path is already packaged and no custom value files + // If the chart at the path is already packaged and no custom values files // options are set, we can copy the chart without making modifications isChartDir := pathIsDir(localRef.Path) - if !isChartDir && len(opts.GetValueFiles()) == 0 { + if !isChartDir && len(opts.GetValuesFiles()) == 0 { if err = copyFileToPath(localRef.Path, p); err != nil { return nil, &BuildError{Reason: ErrChartPull, Err: err} } @@ -103,9 +103,9 @@ func (b *localChartBuilder) Build(ctx context.Context, ref Reference, p string, // Merge chart values, if instructed var mergedValues map[string]interface{} - if len(opts.GetValueFiles()) > 0 { - if mergedValues, err = mergeFileValues(localRef.WorkDir, opts.ValueFiles); err != nil { - return nil, &BuildError{Reason: ErrValueFilesMerge, Err: err} + if len(opts.GetValuesFiles()) > 0 { + if mergedValues, err = mergeFileValues(localRef.WorkDir, opts.ValuesFiles); err != nil { + return nil, &BuildError{Reason: ErrValuesFilesMerge, Err: err} } } @@ -122,9 +122,9 @@ func (b *localChartBuilder) Build(ctx context.Context, ref Reference, p string, // Overwrite default values with merged values, if any if ok, err = OverwriteChartDefaultValues(chart, mergedValues); ok || err != nil { if err != nil { - return nil, &BuildError{Reason: ErrValueFilesMerge, Err: err} + return nil, &BuildError{Reason: ErrValuesFilesMerge, Err: err} } - result.ValueFiles = opts.GetValueFiles() + result.ValuesFiles = opts.GetValuesFiles() } // Ensure dependencies are fetched if building from a directory diff --git a/internal/helm/chart/builder_local_test.go b/internal/helm/chart/builder_local_test.go index 5691371f2..cff5f180f 100644 --- a/internal/helm/chart/builder_local_test.go +++ b/internal/helm/chart/builder_local_test.go @@ -66,7 +66,7 @@ func TestLocalBuilder_Build(t *testing.T) { name string reference Reference buildOpts BuildOptions - valueFiles []helmchart.File + valuesFiles []helmchart.File repositories map[string]*repository.ChartRepository dependentChartPaths []string wantValues chartutil.Values @@ -118,12 +118,12 @@ func TestLocalBuilder_Build(t *testing.T) { wantPackaged: true, }, { - name: "with value files", + name: "with values files", reference: LocalReference{Path: "./../testdata/charts/helmchart"}, buildOpts: BuildOptions{ - ValueFiles: []string{"custom-values1.yaml", "custom-values2.yaml"}, + ValuesFiles: []string{"custom-values1.yaml", "custom-values2.yaml"}, }, - valueFiles: []helmchart.File{ + valuesFiles: []helmchart.File{ { Name: "custom-values1.yaml", Data: []byte(`replicaCount: 11 @@ -189,7 +189,7 @@ fullnameOverride: "full-foo-name-override"`), } // Write value file in the base dir. - for _, f := range tt.valueFiles { + for _, f := range tt.valuesFiles { vPath := filepath.Join(workDir, f.Name) g.Expect(os.WriteFile(vPath, f.Data, 0644)).ToNot(HaveOccurred()) } diff --git a/internal/helm/chart/builder_remote.go b/internal/helm/chart/builder_remote.go index f03c1a8d2..e9cfb9a9e 100644 --- a/internal/helm/chart/builder_remote.go +++ b/internal/helm/chart/builder_remote.go @@ -93,7 +93,7 @@ func (b *remoteChartBuilder) Build(_ context.Context, ref Reference, p string, o if opts.CachedChart != "" && !opts.Force { if curMeta, err := LoadChartMetadataFromArchive(opts.CachedChart); err == nil && result.Version == curMeta.Version { result.Path = opts.CachedChart - result.ValueFiles = opts.GetValueFiles() + result.ValuesFiles = opts.GetValuesFiles() return result, nil } } @@ -105,9 +105,9 @@ func (b *remoteChartBuilder) Build(_ context.Context, ref Reference, p string, o return nil, &BuildError{Reason: ErrChartPull, Err: err} } - // Use literal chart copy from remote if no custom value files options are + // Use literal chart copy from remote if no custom values files options are // set or build option version metadata isn't set. - if len(opts.GetValueFiles()) == 0 && opts.VersionMetadata == "" { + if len(opts.GetValuesFiles()) == 0 && opts.VersionMetadata == "" { if err = validatePackageAndWriteToPath(res, p); err != nil { return nil, &BuildError{Reason: ErrChartPull, Err: err} } @@ -123,17 +123,17 @@ func (b *remoteChartBuilder) Build(_ context.Context, ref Reference, p string, o } chart.Metadata.Version = result.Version - mergedValues, err := mergeChartValues(chart, opts.ValueFiles) + mergedValues, err := mergeChartValues(chart, opts.ValuesFiles) if err != nil { err = fmt.Errorf("failed to merge chart values: %w", err) - return nil, &BuildError{Reason: ErrValueFilesMerge, Err: err} + return nil, &BuildError{Reason: ErrValuesFilesMerge, Err: err} } // Overwrite default values with merged values, if any if ok, err = OverwriteChartDefaultValues(chart, mergedValues); ok || err != nil { if err != nil { - return nil, &BuildError{Reason: ErrValueFilesMerge, Err: err} + return nil, &BuildError{Reason: ErrValuesFilesMerge, Err: err} } - result.ValueFiles = opts.GetValueFiles() + result.ValuesFiles = opts.GetValuesFiles() } // Package the chart with the custom values diff --git a/internal/helm/chart/builder_remote_test.go b/internal/helm/chart/builder_remote_test.go index 80534c60b..a2c33a6fc 100644 --- a/internal/helm/chart/builder_remote_test.go +++ b/internal/helm/chart/builder_remote_test.go @@ -144,7 +144,7 @@ entries: name: "merge values", reference: RemoteReference{Name: "grafana"}, buildOpts: BuildOptions{ - ValueFiles: []string{"a.yaml", "b.yaml", "c.yaml"}, + ValuesFiles: []string{"a.yaml", "b.yaml", "c.yaml"}, }, repository: mockRepo(), wantVersion: "6.17.4", diff --git a/internal/helm/chart/builder_test.go b/internal/helm/chart/builder_test.go index cd64ac41f..d797a209f 100644 --- a/internal/helm/chart/builder_test.go +++ b/internal/helm/chart/builder_test.go @@ -104,29 +104,29 @@ func TestRemoteReference_Validate(t *testing.T) { } } -func TestBuildOptions_GetValueFiles(t *testing.T) { +func TestBuildOptions_GetValuesFiles(t *testing.T) { tests := []struct { - name string - valueFiles []string - want []string + name string + valuesFiles []string + want []string }{ { - name: "Default values.yaml", - valueFiles: []string{chartutil.ValuesfileName}, - want: nil, + name: "Default values.yaml", + valuesFiles: []string{chartutil.ValuesfileName}, + want: nil, }, { - name: "Value files", - valueFiles: []string{chartutil.ValuesfileName, "foo.yaml"}, - want: []string{chartutil.ValuesfileName, "foo.yaml"}, + name: "Values files", + valuesFiles: []string{chartutil.ValuesfileName, "foo.yaml"}, + want: []string{chartutil.ValuesfileName, "foo.yaml"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { g := NewWithT(t) - o := BuildOptions{ValueFiles: tt.valueFiles} - g.Expect(o.GetValueFiles()).To(Equal(tt.want)) + o := BuildOptions{ValuesFiles: tt.valuesFiles} + g.Expect(o.GetValuesFiles()).To(Equal(tt.want)) }) } } @@ -146,14 +146,14 @@ func TestChartBuildResult_Summary(t *testing.T) { want: "Pulled 'chart' chart with version '1.2.3-rc.1+bd6bf40'.", }, { - name: "With value files", + name: "With values files", build: &Build{ - Name: "chart", - Version: "arbitrary-version", - Packaged: true, - ValueFiles: []string{"a.yaml", "b.yaml"}, + Name: "chart", + Version: "arbitrary-version", + Packaged: true, + ValuesFiles: []string{"a.yaml", "b.yaml"}, }, - want: "Packaged 'chart' chart with version 'arbitrary-version', with merged value files [a.yaml b.yaml].", + want: "Packaged 'chart' chart with version 'arbitrary-version', with merged values files [a.yaml b.yaml].", }, { name: "With dependencies", diff --git a/internal/helm/chart/dependency_manager.go b/internal/helm/chart/dependency_manager.go index 798f6df92..e41020655 100644 --- a/internal/helm/chart/dependency_manager.go +++ b/internal/helm/chart/dependency_manager.go @@ -296,7 +296,7 @@ func (dm *DependencyManager) secureLocalChartPath(ref LocalReference, dep *helmc // collectMissing returns a map with reqs that are missing from current, // indexed by their alias or name. All dependencies of a chart are present -// if len of returned value == 0. +// if len of returned map == 0. func collectMissing(current []*helmchart.Chart, reqs []*helmchart.Dependency) map[string]*helmchart.Dependency { // If the number of dependencies equals the number of requested // dependencies, there are no missing dependencies diff --git a/internal/helm/chart/errors.go b/internal/helm/chart/errors.go index 746017f23..696cecc56 100644 --- a/internal/helm/chart/errors.go +++ b/internal/helm/chart/errors.go @@ -59,7 +59,7 @@ func (e *BuildError) Unwrap() error { var ( ErrChartPull = BuildErrorReason("chart pull error") ErrChartMetadataPatch = BuildErrorReason("chart metadata patch error") - ErrValueFilesMerge = BuildErrorReason("value files merge error") + ErrValuesFilesMerge = BuildErrorReason("values files merge error") ErrDependencyBuild = BuildErrorReason("dependency build error") ErrChartPackage = BuildErrorReason("chart package error") ) diff --git a/internal/helm/chart/errors_test.go b/internal/helm/chart/errors_test.go index 7a33c5431..f006f3364 100644 --- a/internal/helm/chart/errors_test.go +++ b/internal/helm/chart/errors_test.go @@ -32,15 +32,15 @@ func TestBuildErrorReason_Error(t *testing.T) { func TestBuildError_Error(t *testing.T) { tests := []struct { - name string - err *BuildError - want string + name string + err *BuildError + want string }{ { name: "with reason", err: &BuildError{ Reason: BuildErrorReason("reason"), - Err: errors.New("error"), + Err: errors.New("error"), }, want: "reason: error", }, @@ -67,7 +67,7 @@ func TestBuildError_Is(t *testing.T) { wrappedErr := errors.New("wrapped") err := &BuildError{ Reason: ErrChartPackage, - Err: wrappedErr, + Err: wrappedErr, } g.Expect(err.Is(ErrChartPackage)).To(BeTrue()) From 4de8f1f862748b2ec181af3e2a6047e09ec72712 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 18 Nov 2021 11:11:30 +0100 Subject: [PATCH 18/23] Allow configuration of Helm file limits This allows custom configuration of the Helm file read limits, allowing a user to overwrite them to their likenings if the defaults are too restrictive for their specific setup using arguments: `--helm-{index,chart,chart-file}-max-size` Signed-off-by: Hidde Beydals --- main.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 55a2d2f97..7853f224b 100644 --- a/main.go +++ b/main.go @@ -45,6 +45,7 @@ import ( sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" "github.com/fluxcd/source-controller/controllers" + "github.com/fluxcd/source-controller/internal/helm" // +kubebuilder:scaffold:imports ) @@ -79,6 +80,9 @@ func main() { concurrent int requeueDependency time.Duration watchAllNamespaces bool + helmIndexLimit int64 + helmChartLimit int64 + helmChartFileLimit int64 clientOptions client.Options logOptions logger.Options leaderElectionOptions leaderelection.Options @@ -98,7 +102,15 @@ func main() { flag.IntVar(&concurrent, "concurrent", 2, "The number of concurrent reconciles per controller.") flag.BoolVar(&watchAllNamespaces, "watch-all-namespaces", true, "Watch for custom resources in all namespaces, if set to false it will only watch the runtime namespace.") - flag.DurationVar(&requeueDependency, "requeue-dependency", 30*time.Second, "The interval at which failing dependencies are reevaluated.") + flag.Int64Var(&helmIndexLimit, "helm-index-max-size", helm.MaxIndexSize, + "The max allowed size in bytes of a Helm repository index file.") + flag.Int64Var(&helmChartLimit, "helm-chart-max-size", helm.MaxChartSize, + "The max allowed size in bytes of a Helm chart file.") + flag.Int64Var(&helmChartFileLimit, "helm-chart-file-max-size", helm.MaxChartFileSize, + "The max allowed size in bytes of a file in a Helm chart.") + flag.DurationVar(&requeueDependency, "requeue-dependency", 30*time.Second, + "The interval at which failing dependencies are reevaluated.") + clientOptions.BindFlags(flag.CommandLine) logOptions.BindFlags(flag.CommandLine) leaderElectionOptions.BindFlags(flag.CommandLine) @@ -106,6 +118,11 @@ func main() { ctrl.SetLogger(logger.NewLogger(logOptions)) + // Set upper bound file size limits Helm + helm.MaxIndexSize = helmIndexLimit + helm.MaxChartSize = helmChartLimit + helm.MaxChartFileSize = helmChartFileLimit + var eventRecorder *events.Recorder if eventsAddr != "" { if er, err := events.NewRecorder(eventsAddr, controllerName); err != nil { From dcd5dd3db1160639c889ec62aa86bbd5be710dc9 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 18 Nov 2021 17:50:35 +0100 Subject: [PATCH 19/23] internal/helm: various nitpicks - Add some more documentation around chart builders - Ensure correct indentation in some doc comments - Provide example of using `errors.Is` for typed `BuildError` - Mention "bytes" in file size limit errors - Add missing copyright header Signed-off-by: Hidde Beydals --- internal/helm/chart/builder_local.go | 30 ++++++++++++++++---- internal/helm/chart/builder_remote.go | 20 +++++++++++-- internal/helm/chart/errors.go | 8 ++++-- internal/helm/chart/metadata.go | 4 +-- internal/helm/repository/chart_repository.go | 2 +- internal/helm/repository/utils_test.go | 16 +++++++++++ 6 files changed, 66 insertions(+), 14 deletions(-) diff --git a/internal/helm/chart/builder_local.go b/internal/helm/chart/builder_local.go index 5d79da3df..a527d3844 100644 --- a/internal/helm/chart/builder_local.go +++ b/internal/helm/chart/builder_local.go @@ -34,16 +34,34 @@ type localChartBuilder struct { dm *DependencyManager } -// NewLocalBuilder returns a Builder capable of building a Helm -// chart with a LocalReference. For chart references pointing to a -// directory, the DependencyManager is used to resolve missing local and -// remote dependencies. +// NewLocalBuilder returns a Builder capable of building a Helm chart with a +// LocalReference. For chart references pointing to a directory, the +// DependencyManager is used to resolve missing local and remote dependencies. func NewLocalBuilder(dm *DependencyManager) Builder { return &localChartBuilder{ dm: dm, } } +// Build attempts to build a Helm chart with the given LocalReference and +// BuildOptions, writing it to p. +// It returns a Build describing the produced (or from cache observed) chart +// written to p, or a BuildError. +// +// The chart is loaded from the LocalReference.Path, and only packaged if the +// version (including BuildOptions.VersionMetadata modifications) differs from +// the current BuildOptions.CachedChart. +// +// BuildOptions.ValuesFiles changes are in this case not taken into account, +// and BuildOptions.Force should be used to enforce a rebuild. +// +// If the LocalReference.Path refers to an already packaged chart, and no +// packaging is required due to BuildOptions modifying the chart, +// LocalReference.Path is copied to p. +// +// If the LocalReference.Path refers to a chart directory, dependencies are +// confirmed to be present using the DependencyManager, while attempting to +// resolve any missing. func (b *localChartBuilder) Build(ctx context.Context, ref Reference, p string, opts BuildOptions) (*Build, error) { localRef, ok := ref.(LocalReference) if !ok { @@ -80,8 +98,8 @@ func (b *localChartBuilder) Build(ctx context.Context, ref Reference, p string, } // If all the following is true, we do not need to package the chart: - // Chart version from metadata matches chart version for ref - // BuildOptions.Force is False + // - Chart version from current metadata matches calculated version + // - BuildOptions.Force is False if opts.CachedChart != "" && !opts.Force { if curMeta, err = LoadChartMetadataFromArchive(opts.CachedChart); err == nil && result.Version == curMeta.Version { result.Path = opts.CachedChart diff --git a/internal/helm/chart/builder_remote.go b/internal/helm/chart/builder_remote.go index e9cfb9a9e..edf1797ae 100644 --- a/internal/helm/chart/builder_remote.go +++ b/internal/helm/chart/builder_remote.go @@ -40,13 +40,27 @@ type remoteChartBuilder struct { } // NewRemoteBuilder returns a Builder capable of building a Helm -// chart with a RemoteReference from the given Index. +// chart with a RemoteReference in the given repository.ChartRepository. func NewRemoteBuilder(repository *repository.ChartRepository) Builder { return &remoteChartBuilder{ remote: repository, } } +// Build attempts to build a Helm chart with the given RemoteReference and +// BuildOptions, writing it to p. +// It returns a Build describing the produced (or from cache observed) chart +// written to p, or a BuildError. +// +// The latest version for the RemoteReference.Version is determined in the +// repository.ChartRepository, only downloading it if the version (including +// BuildOptions.VersionMetadata) differs from the current BuildOptions.CachedChart. +// BuildOptions.ValuesFiles changes are in this case not taken into account, +// and BuildOptions.Force should be used to enforce a rebuild. +// +// After downloading the chart, it is only packaged if required due to BuildOptions +// modifying the chart, otherwise the exact data as retrieved from the repository +// is written to p, after validating it to be a chart. func (b *remoteChartBuilder) Build(_ context.Context, ref Reference, p string, opts BuildOptions) (*Build, error) { remoteRef, ok := ref.(RemoteReference) if !ok { @@ -88,8 +102,8 @@ func (b *remoteChartBuilder) Build(_ context.Context, ref Reference, p string, o } // If all the following is true, we do not need to download and/or build the chart: - // Chart version from metadata matches chart version for ref - // BuildOptions.Force is False + // - Chart version from current metadata matches calculated version + // - BuildOptions.Force is False if opts.CachedChart != "" && !opts.Force { if curMeta, err := LoadChartMetadataFromArchive(opts.CachedChart); err == nil && result.Version == curMeta.Version { result.Path = opts.CachedChart diff --git a/internal/helm/chart/errors.go b/internal/helm/chart/errors.go index 696cecc56..ab074a2b7 100644 --- a/internal/helm/chart/errors.go +++ b/internal/helm/chart/errors.go @@ -35,7 +35,7 @@ type BuildError struct { Err error } -// Error returns Err as a string, prefixed with the Reason, if any. +// Error returns Err as a string, prefixed with the Reason to provide context. func (e *BuildError) Error() string { if e.Reason == nil { return e.Err.Error() @@ -43,7 +43,11 @@ func (e *BuildError) Error() string { return fmt.Sprintf("%s: %s", e.Reason.Error(), e.Err.Error()) } -// Is returns true of the Reason or Err equals target. +// Is returns true if the Reason or Err equals target. +// It can be used to programmatically place an arbitrary Err in the +// context of the Builder: +// err := &BuildError{Reason: ErrChartPull, Err: errors.New("arbitrary transport error")} +// errors.Is(err, ErrChartPull) func (e *BuildError) Is(target error) bool { if e.Reason != nil && e.Reason == target { return true diff --git a/internal/helm/chart/metadata.go b/internal/helm/chart/metadata.go index 24e452089..f59a599b9 100644 --- a/internal/helm/chart/metadata.go +++ b/internal/helm/chart/metadata.go @@ -118,7 +118,7 @@ func LoadChartMetadataFromDir(dir string) (*helmchart.Metadata, error) { return nil, fmt.Errorf("'%s' is a directory", stat.Name()) } if stat.Size() > helm.MaxChartFileSize { - return nil, fmt.Errorf("size of '%s' exceeds '%d' limit", stat.Name(), helm.MaxChartFileSize) + return nil, fmt.Errorf("size of '%s' exceeds '%d' bytes limit", stat.Name(), helm.MaxChartFileSize) } } @@ -145,7 +145,7 @@ func LoadChartMetadataFromArchive(archive string) (*helmchart.Metadata, error) { return nil, err } if stat.Size() > helm.MaxChartSize { - return nil, fmt.Errorf("size of chart '%s' exceeds '%d' limit", stat.Name(), helm.MaxChartSize) + return nil, fmt.Errorf("size of chart '%s' exceeds '%d' bytes limit", stat.Name(), helm.MaxChartSize) } f, err := os.Open(archive) diff --git a/internal/helm/repository/chart_repository.go b/internal/helm/repository/chart_repository.go index c9bab590d..654f55be1 100644 --- a/internal/helm/repository/chart_repository.go +++ b/internal/helm/repository/chart_repository.go @@ -244,7 +244,7 @@ func (r *ChartRepository) LoadFromFile(path string) error { return err } if stat.Size() > helm.MaxIndexSize { - return fmt.Errorf("size of index '%s' exceeds '%d' limit", stat.Name(), helm.MaxIndexSize) + return fmt.Errorf("size of index '%s' exceeds '%d' bytes limit", stat.Name(), helm.MaxIndexSize) } b, err := os.ReadFile(path) if err != nil { diff --git a/internal/helm/repository/utils_test.go b/internal/helm/repository/utils_test.go index fe4cf80ee..bac683b46 100644 --- a/internal/helm/repository/utils_test.go +++ b/internal/helm/repository/utils_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package repository import ( From c202ad59aa86bb1a019a13425fd7139f8ca3b923 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 18 Nov 2021 17:56:16 +0100 Subject: [PATCH 20/23] helm/internal: add `ErrChartReference` This makes it possible to signal reference (validation) errors happening before the build process actually starts dealing with the chart. At present, this does not have a more specific counterpart in the API, but this is expected to change when the conditions logic is revised. Signed-off-by: Hidde Beydals --- internal/helm/chart/builder_local.go | 5 +++-- internal/helm/chart/builder_remote.go | 5 +++-- internal/helm/chart/errors.go | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/helm/chart/builder_local.go b/internal/helm/chart/builder_local.go index a527d3844..ed9b7bafa 100644 --- a/internal/helm/chart/builder_local.go +++ b/internal/helm/chart/builder_local.go @@ -65,11 +65,12 @@ func NewLocalBuilder(dm *DependencyManager) Builder { func (b *localChartBuilder) Build(ctx context.Context, ref Reference, p string, opts BuildOptions) (*Build, error) { localRef, ok := ref.(LocalReference) if !ok { - return nil, fmt.Errorf("expected local chart reference") + err := fmt.Errorf("expected local chart reference") + return nil, &BuildError{Reason: ErrChartReference, Err: err} } if err := ref.Validate(); err != nil { - return nil, &BuildError{Reason: ErrChartPull, Err: err} + return nil, &BuildError{Reason: ErrChartReference, Err: err} } // Load the chart metadata from the LocalReference to ensure it points diff --git a/internal/helm/chart/builder_remote.go b/internal/helm/chart/builder_remote.go index edf1797ae..ab58d0e84 100644 --- a/internal/helm/chart/builder_remote.go +++ b/internal/helm/chart/builder_remote.go @@ -64,11 +64,12 @@ func NewRemoteBuilder(repository *repository.ChartRepository) Builder { func (b *remoteChartBuilder) Build(_ context.Context, ref Reference, p string, opts BuildOptions) (*Build, error) { remoteRef, ok := ref.(RemoteReference) if !ok { - return nil, fmt.Errorf("expected remote chart reference") + err := fmt.Errorf("expected remote chart reference") + return nil, &BuildError{Reason: ErrChartReference, Err: err} } if err := ref.Validate(); err != nil { - return nil, &BuildError{Reason: ErrChartPull, Err: err} + return nil, &BuildError{Reason: ErrChartReference, Err: err} } if err := b.remote.LoadFromCache(); err != nil { diff --git a/internal/helm/chart/errors.go b/internal/helm/chart/errors.go index ab074a2b7..dddd2e298 100644 --- a/internal/helm/chart/errors.go +++ b/internal/helm/chart/errors.go @@ -61,6 +61,7 @@ func (e *BuildError) Unwrap() error { } var ( + ErrChartReference = BuildErrorReason("chart reference error") ErrChartPull = BuildErrorReason("chart pull error") ErrChartMetadataPatch = BuildErrorReason("chart metadata patch error") ErrValuesFilesMerge = BuildErrorReason("values files merge error") From 472eb12f43172c0f38242cf3b08c1db44f0ee46c Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 18 Nov 2021 20:40:44 +0100 Subject: [PATCH 21/23] controllers: set generation as version metadata By providing the Generation of the object that is getting reconciled as version metadata to the builder if any custom values files are defined, the Artifact revision changes if the specification does, ensuring consumers of the Artifact are able to react to changes in values (and perform a release). Signed-off-by: Hidde Beydals --- controllers/helmchart_controller.go | 52 ++++++++++++++++++----------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/controllers/helmchart_controller.go b/controllers/helmchart_controller.go index 685a43a57..d6c46137a 100644 --- a/controllers/helmchart_controller.go +++ b/controllers/helmchart_controller.go @@ -23,6 +23,7 @@ import ( "net/url" "os" "path/filepath" + "strconv" "strings" "time" @@ -332,24 +333,29 @@ func (r *HelmChartReconciler) fromHelmRepository(ctx context.Context, repo sourc } // Build the chart - cBuilder := chart.NewRemoteBuilder(chartRepo) + cb := chart.NewRemoteBuilder(chartRepo) ref := chart.RemoteReference{Name: c.Spec.Chart, Version: c.Spec.Version} opts := chart.BuildOptions{ ValuesFiles: c.GetValuesFiles(), CachedChart: cachedChart, Force: force, } - build, err := cBuilder.Build(ctx, ref, filepath.Join(workDir, "chart.tgz"), opts) + // Set the VersionMetadata to the object's Generation if ValuesFiles is defined + // This ensures changes can be noticed by the Artifact consumer + if len(opts.GetValuesFiles()) > 0 { + opts.VersionMetadata = strconv.FormatInt(c.Generation, 10) + } + b, err := cb.Build(ctx, ref, filepath.Join(workDir, "chart.tgz"), opts) if err != nil { return sourcev1.HelmChartNotReady(c, sourcev1.ChartPullFailedReason, err.Error()), err } - newArtifact := r.Storage.NewArtifactFor(c.Kind, c.GetObjectMeta(), build.Version, - fmt.Sprintf("%s-%s.tgz", build.Name, build.Version)) + newArtifact := r.Storage.NewArtifactFor(c.Kind, c.GetObjectMeta(), b.Version, + fmt.Sprintf("%s-%s.tgz", b.Name, b.Version)) // If the path of the returned build equals the cache path, // there are no changes to the chart - if build.Path == cachedChart { + if b.Path == cachedChart { // Ensure hostname is updated if c.GetArtifact().URL != newArtifact.URL { r.Storage.SetArtifactURL(c.GetArtifact()) @@ -374,18 +380,18 @@ func (r *HelmChartReconciler) fromHelmRepository(ctx context.Context, repo sourc defer unlock() // Copy the packaged chart to the artifact path - if err = r.Storage.CopyFromPath(&newArtifact, build.Path); err != nil { + if err = r.Storage.CopyFromPath(&newArtifact, b.Path); err != nil { err = fmt.Errorf("failed to write chart package to storage: %w", err) return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } // Update symlink - cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", build.Name)) + cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", b.Name)) if err != nil { err = fmt.Errorf("storage error: %w", err) return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } - return sourcev1.HelmChartReady(c, newArtifact, cUrl, sourcev1.ChartPullSucceededReason, build.Summary()), nil + return sourcev1.HelmChartReady(c, newArtifact, cUrl, sourcev1.ChartPullSucceededReason, b.Summary()), nil } func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source sourcev1.Artifact, c sourcev1.HelmChart, @@ -430,35 +436,43 @@ func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source so defer dm.Clear() // Configure builder options, including any previously cached chart - buildsOpts := chart.BuildOptions{ + opts := chart.BuildOptions{ ValuesFiles: c.GetValuesFiles(), Force: force, } if artifact := c.Status.Artifact; artifact != nil { - buildsOpts.CachedChart = artifact.Path + opts.CachedChart = artifact.Path } // Add revision metadata to chart build if c.Spec.ReconcileStrategy == sourcev1.ReconcileStrategyRevision { // Isolate the commit SHA from GitRepository type artifacts by removing the branch/ prefix. splitRev := strings.Split(source.Revision, "/") - buildsOpts.VersionMetadata = splitRev[len(splitRev)-1] + opts.VersionMetadata = splitRev[len(splitRev)-1] + } + // Set the VersionMetadata to the object's Generation if ValuesFiles is defined + // This ensures changes can be noticed by the Artifact consumer + if len(opts.GetValuesFiles()) > 0 { + if opts.VersionMetadata != "" { + opts.VersionMetadata += "." + } + opts.VersionMetadata += strconv.FormatInt(c.Generation, 10) } // Build chart - chartBuilder := chart.NewLocalBuilder(dm) - result, err := chartBuilder.Build(ctx, chart.LocalReference{WorkDir: sourceDir, Path: chartPath}, filepath.Join(workDir, "chart.tgz"), buildsOpts) + cb := chart.NewLocalBuilder(dm) + b, err := cb.Build(ctx, chart.LocalReference{WorkDir: sourceDir, Path: chartPath}, filepath.Join(workDir, "chart.tgz"), opts) if err != nil { return sourcev1.HelmChartNotReady(c, reasonForBuildError(err), err.Error()), err } - newArtifact := r.Storage.NewArtifactFor(c.Kind, c.GetObjectMeta(), result.Version, - fmt.Sprintf("%s-%s.tgz", result.Name, result.Version)) + newArtifact := r.Storage.NewArtifactFor(c.Kind, c.GetObjectMeta(), b.Version, + fmt.Sprintf("%s-%s.tgz", b.Name, b.Version)) // If the path of the returned build equals the cache path, // there are no changes to the chart if apimeta.IsStatusConditionTrue(c.Status.Conditions, meta.ReadyCondition) && - result.Path == buildsOpts.CachedChart { + b.Path == opts.CachedChart { // Ensure hostname is updated if c.GetArtifact().URL != newArtifact.URL { r.Storage.SetArtifactURL(c.GetArtifact()) @@ -483,19 +497,19 @@ func (r *HelmChartReconciler) fromTarballArtifact(ctx context.Context, source so defer unlock() // Copy the packaged chart to the artifact path - if err = r.Storage.CopyFromPath(&newArtifact, result.Path); err != nil { + if err = r.Storage.CopyFromPath(&newArtifact, b.Path); err != nil { err = fmt.Errorf("failed to write chart package to storage: %w", err) return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } // Update symlink - cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", result.Name)) + cUrl, err := r.Storage.Symlink(newArtifact, fmt.Sprintf("%s-latest.tgz", b.Name)) if err != nil { err = fmt.Errorf("storage error: %w", err) return sourcev1.HelmChartNotReady(c, sourcev1.StorageOperationFailedReason, err.Error()), err } - return sourcev1.HelmChartReady(c, newArtifact, cUrl, reasonForBuildSuccess(result), result.Summary()), nil + return sourcev1.HelmChartReady(c, newArtifact, cUrl, reasonForBuildSuccess(b), b.Summary()), nil } // namespacedChartRepositoryCallback returns a chart.GetChartRepositoryCallback From 88ff049ab02cd268d55c32c373706f9e47fb93db Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 18 Nov 2021 21:04:56 +0100 Subject: [PATCH 22/23] internal/helm: ensure cached chart name matches This helps detect e.g. path or chart name reference changes. Signed-off-by: Hidde Beydals --- internal/helm/chart/builder_local.go | 13 ++++++++----- internal/helm/chart/builder_remote.go | 13 ++++++++----- internal/helm/chart/builder_remote_test.go | 7 ++++--- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/internal/helm/chart/builder_local.go b/internal/helm/chart/builder_local.go index ed9b7bafa..963588815 100644 --- a/internal/helm/chart/builder_local.go +++ b/internal/helm/chart/builder_local.go @@ -99,13 +99,16 @@ func (b *localChartBuilder) Build(ctx context.Context, ref Reference, p string, } // If all the following is true, we do not need to package the chart: - // - Chart version from current metadata matches calculated version + // - Chart name from cached chart matches resolved name + // - Chart version from cached chart matches calculated version // - BuildOptions.Force is False if opts.CachedChart != "" && !opts.Force { - if curMeta, err = LoadChartMetadataFromArchive(opts.CachedChart); err == nil && result.Version == curMeta.Version { - result.Path = opts.CachedChart - result.ValuesFiles = opts.ValuesFiles - return result, nil + if curMeta, err = LoadChartMetadataFromArchive(opts.CachedChart); err == nil { + if result.Name == curMeta.Name && result.Version == curMeta.Version { + result.Path = opts.CachedChart + result.ValuesFiles = opts.ValuesFiles + return result, nil + } } } diff --git a/internal/helm/chart/builder_remote.go b/internal/helm/chart/builder_remote.go index ab58d0e84..617e2ec5e 100644 --- a/internal/helm/chart/builder_remote.go +++ b/internal/helm/chart/builder_remote.go @@ -103,13 +103,16 @@ func (b *remoteChartBuilder) Build(_ context.Context, ref Reference, p string, o } // If all the following is true, we do not need to download and/or build the chart: - // - Chart version from current metadata matches calculated version + // - Chart name from cached chart matches resolved name + // - Chart version from cached chart matches calculated version // - BuildOptions.Force is False if opts.CachedChart != "" && !opts.Force { - if curMeta, err := LoadChartMetadataFromArchive(opts.CachedChart); err == nil && result.Version == curMeta.Version { - result.Path = opts.CachedChart - result.ValuesFiles = opts.GetValuesFiles() - return result, nil + if curMeta, err := LoadChartMetadataFromArchive(opts.CachedChart); err == nil { + if result.Name == curMeta.Name && result.Version == curMeta.Version { + result.Path = opts.CachedChart + result.ValuesFiles = opts.GetValuesFiles() + return result, nil + } } } diff --git a/internal/helm/chart/builder_remote_test.go b/internal/helm/chart/builder_remote_test.go index a2c33a6fc..56c1fd855 100644 --- a/internal/helm/chart/builder_remote_test.go +++ b/internal/helm/chart/builder_remote_test.go @@ -207,11 +207,12 @@ func TestRemoteBuilder_Build_CachedChart(t *testing.T) { index := []byte(` apiVersion: v1 entries: - grafana: + helmchart: - urls: - - https://example.com/grafana.tgz + - https://example.com/helmchart-0.1.0.tgz description: string version: 0.1.0 + name: helmchart `) mockGetter := &mockIndexChartGetter{ @@ -226,7 +227,7 @@ entries: } } - reference := RemoteReference{Name: "grafana"} + reference := RemoteReference{Name: "helmchart"} repository := mockRepo() _, err = repository.CacheIndex() From 2392326ba9c3f3ca04f181b1325f3a9f5c00995d Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Fri, 19 Nov 2021 17:13:01 +0100 Subject: [PATCH 23/23] internal/helm: doc block nitpicks Signed-off-by: Hidde Beydals --- internal/helm/chart/dependency_manager.go | 17 ++++++++++------- internal/helm/chart/metadata_test.go | 2 +- internal/helm/repository/chart_repository.go | 4 ++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/internal/helm/chart/dependency_manager.go b/internal/helm/chart/dependency_manager.go index e41020655..1a053e623 100644 --- a/internal/helm/chart/dependency_manager.go +++ b/internal/helm/chart/dependency_manager.go @@ -41,13 +41,14 @@ type GetChartRepositoryCallback func(url string) (*repository.ChartRepository, e // DependencyManager manages dependencies for a Helm chart. type DependencyManager struct { - // repositories contains a map of Index indexed by their - // normalized URL. It is used as a lookup table for missing - // dependencies. + // repositories contains a map of repository.ChartRepository objects + // indexed by their repository.NormalizeURL. + // It is consulted as a lookup table for missing dependencies, based on + // the (repository) URL the dependency refers to. repositories map[string]*repository.ChartRepository // getRepositoryCallback can be set to an on-demand GetChartRepositoryCallback - // which returned result is cached to repositories. + // whose returned result is cached to repositories. getRepositoryCallback GetChartRepositoryCallback // concurrent is the number of concurrent chart-add operations during @@ -91,6 +92,8 @@ func NewDependencyManager(opts ...DependencyManagerOption) *DependencyManager { return dm } +// Clear iterates over the repositories, calling Unload and RemoveCache on all +// items. It returns a collection of (cache removal) errors. func (dm *DependencyManager) Clear() []error { var errs []error for _, v := range dm.repositories { @@ -294,9 +297,9 @@ func (dm *DependencyManager) secureLocalChartPath(ref LocalReference, dep *helmc return securejoin.SecureJoin(ref.WorkDir, filepath.Join(relPath, localUrl.Host, localUrl.Path)) } -// collectMissing returns a map with reqs that are missing from current, -// indexed by their alias or name. All dependencies of a chart are present -// if len of returned map == 0. +// collectMissing returns a map with dependencies from reqs that are missing +// from current, indexed by their alias or name. All dependencies of a chart +// are present if len of returned map == 0. func collectMissing(current []*helmchart.Chart, reqs []*helmchart.Dependency) map[string]*helmchart.Dependency { // If the number of dependencies equals the number of requested // dependencies, there are no missing dependencies diff --git a/internal/helm/chart/metadata_test.go b/internal/helm/chart/metadata_test.go index d9c882f43..07449100a 100644 --- a/internal/helm/chart/metadata_test.go +++ b/internal/helm/chart/metadata_test.go @@ -1,5 +1,5 @@ /* -Copyright 2020 The Flux authors +Copyright 2021 The Flux authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/internal/helm/repository/chart_repository.go b/internal/helm/repository/chart_repository.go index 654f55be1..8cee2e026 100644 --- a/internal/helm/repository/chart_repository.go +++ b/internal/helm/repository/chart_repository.go @@ -285,7 +285,7 @@ func (r *ChartRepository) CacheIndex() (string, error) { // StrategicallyLoadIndex lazy-loads the Index from CachePath using // LoadFromCache if it does not HasIndex. -// If it not HasCacheFile, a cache attempt is made using CacheIndex +// If not HasCacheFile, a cache attempt is made using CacheIndex // before continuing to load. func (r *ChartRepository) StrategicallyLoadIndex() (err error) { if r.HasIndex() { @@ -350,7 +350,7 @@ func (r *ChartRepository) HasCacheFile() bool { } // Unload can be used to signal the Go garbage collector the Index can -// be freed from memory if the Index object is expected to +// be freed from memory if the ChartRepository object is expected to // continue to exist in the stack for some time. func (r *ChartRepository) Unload() { if r == nil {