Add persistent storage for Flux artifacts

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>

README.md

@@ -119,6 +119,9 @@ spec:
     multitenant: true
     networkPolicy: true
     domain: "cluster.local"
+  storage:
+    class: "standard"
+    size: "10Gi"
 ```
 
 Every hour, the operator will check for updates in the ControlPlane

api/v1/fluxinstance_types.go

@@ -44,6 +44,11 @@ type FluxInstanceSpec struct {
 	// +optional
 	Cluster *Cluster `json:"cluster,omitempty"`
 
+	// Storage holds the specification of the source-controller
+	// persistent volume claim.
+	// +optional
+	Storage *Storage `json:"storage,omitempty"`
+
 	// Kustomize holds a set of patches that can be applied to the
 	// Flux installation, to customize the way Flux operates.
 	// +optional
@@ -122,6 +127,17 @@ type Cluster struct {
 	Type string `json:"type,omitempty"`
 }
 
+// Storage is the specification for the persistent volume claim.
+type Storage struct {
+	// Class is the storage class to use for the PVC.
+	// +required
+	Class string `json:"class"`
+
+	// Size is the size of the PVC.
+	// +required
+	Size string `json:"size"`
+}
+
 // Kustomize holds a set of patches that can be applied to the
 // Flux installation, to customize the way Flux operates.
 type Kustomize struct {

config/crd/bases/fluxcd.controlplane.io_fluxinstances.yaml

@@ -185,6 +185,21 @@ spec:
                       type: object
                     type: array
                 type: object
+              storage:
+                description: |-
+                  Storage holds the specification of the source-controller
+                  persistent volume claim.
+                properties:
+                  class:
+                    description: Class is the storage class to use for the PVC.
+                    type: string
+                  size:
+                    description: Size is the size of the PVC.
+                    type: string
+                required:
+                - class
+                - size
+                type: object
               wait:
                 default: true
                 description: |-

config/samples/fluxcd_v1_fluxinstance.yaml

@@ -18,6 +18,9 @@ spec:
     multitenant: false
     networkPolicy: true
     domain: "cluster.local"
+  storage:
+    class: "standard"
+    size: "1Gi"
   kustomize:
     patches:
       - target:

internal/builder/build.go

@@ -74,6 +74,12 @@ func generate(base string, options Options) error {
 		return fmt.Errorf("generate node selector failed: %w", err)
 	}
 
+	if options.ArtifactStorage != nil {
+		if err := execTemplate(options, pvcTmpl, path.Join(base, "pvc.yaml")); err != nil {
+			return fmt.Errorf("generate pvc failed: %w", err)
+		}
+	}
+
 	if err := execTemplate(options, kustomizationTmpl, path.Join(base, "kustomization.yaml")); err != nil {
 		return fmt.Errorf("generate kustomization failed: %w", err)
 	}

internal/builder/build_test.go

@@ -15,6 +15,7 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
+//nolint:goconst
 func TestBuild_Defaults(t *testing.T) {
 	g := NewWithT(t)
 	const version = "v2.3.0"
@@ -49,6 +50,7 @@ func TestBuild_Defaults(t *testing.T) {
 	g.Expect(string(genK)).To(Equal(string(goldenK)))
 }
 
+//nolint:goconst
 func TestBuild_Patches(t *testing.T) {
 	g := NewWithT(t)
 	const version = "v2.3.0"
@@ -113,6 +115,7 @@ func TestBuild_Patches(t *testing.T) {
 	g.Expect(found).To(BeTrue())
 }
 
+//nolint:goconst
 func TestBuild_Profiles(t *testing.T) {
 	g := NewWithT(t)
 	const version = "v2.3.0"
@@ -136,10 +139,10 @@ func TestBuild_Profiles(t *testing.T) {
 	g.Expect(result.Objects).NotTo(BeEmpty())
 	g.Expect(result.Revision).To(HavePrefix(version + "@sha256:"))
 
-	//if os.Getenv("GEN_GOLDEN") == "true" {
-	err = cp.Copy(filepath.Join(dstDir, "kustomization.yaml"), goldenFile)
-	g.Expect(err).NotTo(HaveOccurred())
-	//}
+	if os.Getenv("GEN_GOLDEN") == "true" {
+		err = cp.Copy(filepath.Join(dstDir, "kustomization.yaml"), goldenFile)
+		g.Expect(err).NotTo(HaveOccurred())
+	}
 
 	genK, err := os.ReadFile(filepath.Join(dstDir, "kustomization.yaml"))
 	g.Expect(err).NotTo(HaveOccurred())
@@ -161,6 +164,57 @@ func TestBuild_Profiles(t *testing.T) {
 	g.Expect(found).To(BeTrue())
 }
 
+//nolint:goconst
+func TestBuild_ArtifactStorage(t *testing.T) {
+	g := NewWithT(t)
+	const version = "v2.3.0"
+	options := MakeDefaultOptions()
+	options.Version = version
+
+	srcDir := filepath.Join("testdata", version)
+	goldenFile := filepath.Join("testdata", version+"-golden", "storage.kustomization.yaml")
+
+	dstDir, err := testTempDir(t)
+	g.Expect(err).NotTo(HaveOccurred())
+
+	ci, err := ExtractComponentImages(srcDir, options)
+	g.Expect(err).NotTo(HaveOccurred())
+	options.ComponentImages = ci
+
+	options.ArtifactStorage = &ArtifactStorage{
+		Class: "standard",
+		Size:  "10Gi",
+	}
+
+	result, err := Build(srcDir, dstDir, options)
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(result.Objects).NotTo(BeEmpty())
+	g.Expect(result.Revision).To(HavePrefix(version + "@sha256:"))
+
+	if os.Getenv("GEN_GOLDEN") == "true" {
+		err = cp.Copy(filepath.Join(dstDir, "kustomization.yaml"), goldenFile)
+		g.Expect(err).NotTo(HaveOccurred())
+	}
+
+	genK, err := os.ReadFile(filepath.Join(dstDir, "kustomization.yaml"))
+	g.Expect(err).NotTo(HaveOccurred())
+
+	goldenK, err := os.ReadFile(goldenFile)
+	g.Expect(err).NotTo(HaveOccurred())
+
+	g.Expect(string(genK)).To(Equal(string(goldenK)))
+
+	found := false
+	for _, obj := range result.Objects {
+		if obj.GetKind() == "PersistentVolumeClaim" {
+			found = true
+			g.Expect(obj.GetName()).To(Equal("source-controller"))
+		}
+	}
+	g.Expect(found).To(BeTrue())
+}
+
+//nolint:goconst
 func TestBuild_InvalidPatches(t *testing.T) {
 	g := NewWithT(t)
 	const version = "v2.3.0"

internal/builder/images.go

@@ -21,14 +21,6 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
-// ComponentImage represents a container image used by a component.
-type ComponentImage struct {
-	Name       string
-	Repository string
-	Tag        string
-	Digest     string
-}
-
 // ExtractComponentImages reads the source directory and extracts the container images
 // from the components manifests.
 func ExtractComponentImages(srcDir string, opts Options) ([]ComponentImage, error) {

internal/builder/options.go

@@ -19,6 +19,7 @@ type Options struct {
 	ClusterDomain          string
 	TolerationKeys         []string
 	Patches                string
+	ArtifactStorage        *ArtifactStorage
 }
 
 // MakeDefaultOptions returns the default builder configuration.
@@ -44,3 +45,17 @@ func MakeDefaultOptions() Options {
 		ClusterDomain:          "cluster.local",
 	}
 }
+
+// ComponentImage represents a container image used by a component.
+type ComponentImage struct {
+	Name       string
+	Repository string
+	Tag        string
+	Digest     string
+}
+
+// ArtifactStorage represents the source-controller PVC.
+type ArtifactStorage struct {
+	Class string
+	Size  string
+}

internal/builder/templates.go

@@ -17,9 +17,11 @@ var kustomizationTmpl = `---
 {{- $registry := .Registry }}
 {{- $logLevel := .LogLevel }}
 {{- $clusterDomain := .ClusterDomain }}
+{{- $artifactStorage := .ArtifactStorage }}
+{{- $namespace := .Namespace }}
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-namespace: {{.Namespace}}
+namespace: {{$namespace}}
 transformers:
   - labels.yaml
 resources:
@@ -31,6 +33,9 @@ resources:
 {{- range .Components }}
   - {{.}}.yaml
 {{- end }}
+{{- if $artifactStorage }}
+  - pvc.yaml
+{{- end }}
 {{- if $registry }}
 images:
 {{- range .ComponentImages }}
@@ -79,6 +84,19 @@ patches:
     - op: replace
       path: /spec/template/spec/containers/0/args/6
       value: --storage-adv-addr=source-controller.$(RUNTIME_NAMESPACE).svc.{{$clusterDomain}}.
+{{- if $artifactStorage }}
+    - op: add
+      path: '/spec/template/spec/volumes/-'
+      value:
+        name: persistent-data
+        persistentVolumeClaim:
+          claimName: source-controller
+    - op: replace
+      path: '/spec/template/spec/containers/0/volumeMounts/0'
+      value:
+        name: persistent-data
+        mountPath: /data
+{{- end }}
 {{- else }}
 - target:
     group: apps
@@ -159,6 +177,20 @@ metadata:
     fluxcd.controlplane.io/prune: disabled
 `
 
+var pvcTmpl = `---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: source-controller
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: {{.ArtifactStorage.Class}}
+  resources:
+    requests:
+      storage: {{.ArtifactStorage.Size}}
+`
+
 func execTemplate(obj interface{}, tmpl, filename string) (err error) {
 	t, err := template.New("tmpl").Parse(tmpl)
 	if err != nil {