Add include property to GitRepositories

Signed-off-by: Philip Laine <philip.laine@gmail.com>

api/v1beta1/gitrepository_types.go

@@ -87,6 +87,34 @@ type GitRepositorySpec struct {
 	// This option is available only when using the 'go-git' GitImplementation.
 	// +optional
 	RecurseSubmodules bool `json:"recurseSubmodules,omitempty"`
+
+	// Extra git repositories to map into the repository
+	Include []GitRepositoryInclude `json:"include,omitempty"`
+}
+
+func (in *GitRepositoryInclude) GetFromPath() string {
+	return in.FromPath
+}
+
+func (in *GitRepositoryInclude) GetToPath() string {
+	if in.ToPath == "" {
+		return in.GitRepositoryRef.Name
+	}
+	return in.ToPath
+}
+
+// GitRepositoryInclude defines a source with a from and to path.
+type GitRepositoryInclude struct {
+	// Reference to a GitRepository to include.
+	GitRepositoryRef meta.LocalObjectReference `json:"repository"`
+
+	// The path to copy contents from, defaults to the root directory.
+	// +optional
+	FromPath string `json:"fromPath"`
+
+	// The path to copy contents to, defaults to the name of the source ref.
+	// +optional
+	ToPath string `json:"toPath"`
 }
 
 // GitRepositoryRef defines the Git ref used for pull and checkout operations.
@@ -138,6 +166,10 @@ type GitRepositoryStatus struct {
 	// +optional
 	Artifact *Artifact `json:"artifact,omitempty"`
 
+	// IncludeArtifacts represents the included artifacts from the last successful repository sync.
+	// +optional
+	IncludeArtifacts []*Artifact `json:"includeArtifacts,omitempty"`
+
 	meta.ReconcileRequestStatus `json:",inline"`
 }
 
@@ -166,8 +198,9 @@ func GitRepositoryProgressing(repository GitRepository) GitRepository {
 // GitRepositoryReady sets the given Artifact and URL on the GitRepository and
 // sets the meta.ReadyCondition to 'True', with the given reason and message. It
 // returns the modified GitRepository.
-func GitRepositoryReady(repository GitRepository, artifact Artifact, url, reason, message string) GitRepository {
+func GitRepositoryReady(repository GitRepository, artifact Artifact, includeArtifacts []*Artifact, url, reason, message string) GitRepository {
 	repository.Status.Artifact = &artifact
+	repository.Status.IncludeArtifacts = includeArtifacts
 	repository.Status.URL = url
 	meta.SetResourceCondition(&repository, meta.ReadyCondition, metav1.ConditionTrue, reason, message)
 	return repository

config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml

@@ -65,6 +65,33 @@ spec:
                   a default will be used, consult the documentation for your version
                   to find out what those are.
                 type: string
+              include:
+                description: Extra git repositories to map into the repository
+                items:
+                  description: GitRepositoryInclude defines a source with a from and
+                    to path.
+                  properties:
+                    fromPath:
+                      description: The path to copy contents from, defaults to the
+                        root directory.
+                      type: string
+                    repository:
+                      description: Reference to a GitRepository to include.
+                      properties:
+                        name:
+                          description: Name of the referent
+                          type: string
+                      required:
+                      - name
+                      type: object
+                    toPath:
+                      description: The path to copy contents to, defaults to the name
+                        of the source ref.
+                      type: string
+                  required:
+                  - repository
+                  type: object
+                type: array
               interval:
                 description: The interval at which to check for repository updates.
                 type: string
@@ -245,6 +272,36 @@ spec:
                   - type
                   type: object
                 type: array
+              includeArtifacts:
+                description: IncludeArtifacts represents the included artifacts from
+                  the last successful repository sync.
+                items:
+                  description: Artifact represents the output of a source synchronisation.
+                  properties:
+                    checksum:
+                      description: Checksum is the SHA1 checksum of the artifact.
+                      type: string
+                    lastUpdateTime:
+                      description: LastUpdateTime is the timestamp corresponding to
+                        the last update of this artifact.
+                      format: date-time
+                      type: string
+                    path:
+                      description: Path is the relative file path of this artifact.
+                      type: string
+                    revision:
+                      description: Revision is a human readable identifier traceable
+                        in the origin source system. It can be a Git commit SHA, Git
+                        tag, a Helm index timestamp, a Helm chart version, etc.
+                      type: string
+                    url:
+                      description: URL is the HTTP address of this artifact.
+                      type: string
+                  required:
+                  - path
+                  - url
+                  type: object
+                type: array
               lastHandledReconcileAt:
                 description: LastHandledReconcileAt holds the value of the most recent
                   reconcile request value, so a change can be detected.

controllers/gitrepository_controller.go

@@ -34,6 +34,7 @@ import (
 	kuberecorder "k8s.io/client-go/tools/record"
 	"k8s.io/client-go/tools/reference"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -58,6 +59,7 @@ import (
 // GitRepositoryReconciler reconciles a GitRepository object
 type GitRepositoryReconciler struct {
 	client.Client
+	requeueDependency     time.Duration
 	Scheme                *runtime.Scheme
 	Storage               *Storage
 	EventRecorder         kuberecorder.EventRecorder
@@ -66,17 +68,21 @@ type GitRepositoryReconciler struct {
 }
 
 type GitRepositoryReconcilerOptions struct {
-	MaxConcurrentReconciles int
+	MaxConcurrentReconciles   int
+	DependencyRequeueInterval time.Duration
 }
 
 func (r *GitRepositoryReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return r.SetupWithManagerAndOptions(mgr, GitRepositoryReconcilerOptions{})
 }
 
 func (r *GitRepositoryReconciler) SetupWithManagerAndOptions(mgr ctrl.Manager, opts GitRepositoryReconcilerOptions) error {
+	r.requeueDependency = opts.DependencyRequeueInterval
+
 	return ctrl.NewControllerManagedBy(mgr).
-		For(&sourcev1.GitRepository{}).
-		WithEventFilter(predicate.Or(predicate.GenerationChangedPredicate{}, predicates.ReconcileRequestedPredicate{})).
+		For(&sourcev1.GitRepository{}, builder.WithPredicates(
+			predicate.Or(predicate.GenerationChangedPredicate{}, predicates.ReconcileRequestedPredicate{}),
+		)).
 		WithOptions(controller.Options{MaxConcurrentReconciles: opts.MaxConcurrentReconciles}).
 		Complete(r)
 }
@@ -113,6 +119,25 @@ func (r *GitRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		return ctrl.Result{}, nil
 	}
 
+	// check dependencies
+	if len(repository.Spec.Include) > 0 {
+		if err := r.checkDependencies(repository); err != nil {
+			repository = sourcev1.GitRepositoryNotReady(repository, meta.DependencyNotReadyReason, err.Error())
+			if err := r.updateStatus(ctx, req, repository.Status); err != nil {
+				log.Error(err, "unable to update status for dependency not ready")
+				return ctrl.Result{Requeue: true}, err
+			}
+			// we can't rely on exponential backoff because it will prolong the execution too much,
+			// instead we requeue on a fix interval.
+			msg := fmt.Sprintf("Dependencies do not meet ready condition, retrying in %s", r.requeueDependency.String())
+			log.Info(msg)
+			r.event(ctx, repository, events.EventSeverityInfo, msg)
+			r.recordReadiness(ctx, repository)
+			return ctrl.Result{RequeueAfter: r.requeueDependency}, nil
+		}
+		log.Info("All dependencies area ready, proceeding with reconciliation")
+	}
+
 	// record reconciliation duration
 	if r.MetricsRecorder != nil {
 		objRef, err := reference.GetReference(r.Scheme, &repository)
@@ -174,6 +199,27 @@ func (r *GitRepositoryReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 	return ctrl.Result{RequeueAfter: repository.GetInterval().Duration}, nil
 }
 
+func (r *GitRepositoryReconciler) checkDependencies(repository sourcev1.GitRepository) error {
+	for _, d := range repository.Spec.Include {
+		dName := types.NamespacedName{Name: d.GitRepositoryRef.Name, Namespace: repository.Namespace}
+		var gr sourcev1.GitRepository
+		err := r.Get(context.Background(), dName, &gr)
+		if err != nil {
+			return fmt.Errorf("unable to get '%s' dependency: %w", dName, err)
+		}
+
+		if len(gr.Status.Conditions) == 0 || gr.Generation != gr.Status.ObservedGeneration {
+			return fmt.Errorf("dependency '%s' is not ready", dName)
+		}
+
+		if !apimeta.IsStatusConditionTrue(gr.Status.Conditions, meta.ReadyCondition) {
+			return fmt.Errorf("dependency '%s' is not ready", dName)
+		}
+	}
+
+	return nil
+}
+
 func (r *GitRepositoryReconciler) reconcile(ctx context.Context, repository sourcev1.GitRepository) (sourcev1.GitRepository, error) {
 	// create tmp dir for the Git clone
 	tmpGit, err := ioutil.TempDir("", repository.Name)
@@ -220,18 +266,36 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, repository sour
 		git.CheckoutOptions{
 			GitImplementation: repository.Spec.GitImplementation,
 			RecurseSubmodules: repository.Spec.RecurseSubmodules,
-		})
+		},
+	)
 	if err != nil {
 		return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
 	}
 	commit, revision, err := checkoutStrategy.Checkout(ctx, tmpGit, repository.Spec.URL, auth)
 	if err != nil {
 		return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
 	}
-
-	// return early on unchanged revision
 	artifact := r.Storage.NewArtifactFor(repository.Kind, repository.GetObjectMeta(), revision, fmt.Sprintf("%s.tar.gz", commit.Hash()))
-	if apimeta.IsStatusConditionTrue(repository.Status.Conditions, meta.ReadyCondition) && repository.GetArtifact().HasRevision(artifact.Revision) {
+
+	// copy all included repository into the artifact
+	includeArtifacts := []*sourcev1.Artifact{}
+	for _, incl := range repository.Spec.Include {
+		dName := types.NamespacedName{Name: incl.GitRepositoryRef.Name, Namespace: repository.Namespace}
+		var gr sourcev1.GitRepository
+		err := r.Get(context.Background(), dName, &gr)
+		if err != nil {
+			return sourcev1.GitRepositoryNotReady(repository, meta.DependencyNotReadyReason, err.Error()), err
+		}
+		includeArtifacts = append(includeArtifacts, gr.GetArtifact())
+
+		err = r.Storage.CopyToPath(gr.GetArtifact(), incl.GetFromPath(), filepath.Join(tmpGit, incl.GetToPath()))
+		if err != nil {
+			return sourcev1.GitRepositoryNotReady(repository, meta.DependencyNotReadyReason, err.Error()), err
+		}
+	}
+
+	// return early on unchanged revision and unchanged included repositories
+	if apimeta.IsStatusConditionTrue(repository.Status.Conditions, meta.ReadyCondition) && repository.GetArtifact().HasRevision(artifact.Revision) && !hasArtifactUpdated(repository.Status.IncludeArtifacts, includeArtifacts) {
 		if artifact.URL != repository.GetArtifact().URL {
 			r.Storage.SetArtifactURL(repository.GetArtifact())
 			repository.Status.URL = r.Storage.SetHostname(repository.Status.URL)
@@ -295,7 +359,26 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, repository sour
 	}
 
 	message := fmt.Sprintf("Fetched revision: %s", artifact.Revision)
-	return sourcev1.GitRepositoryReady(repository, artifact, url, sourcev1.GitOperationSucceedReason, message), nil
+	return sourcev1.GitRepositoryReady(repository, artifact, includeArtifacts, url, sourcev1.GitOperationSucceedReason, message), nil
+}
+
+// hasArtifactUpdated returns true if any of the revisions in the current artifacts
+// does not match any of the artifacts in the updated artifacts
+func hasArtifactUpdated(current []*sourcev1.Artifact, updated []*sourcev1.Artifact) bool {
+	if len(current) != len(updated) {
+		return true
+	}
+
+	for _, c := range current {
+		for _, u := range updated {
+			if u.HasRevision(c.Revision) {
+				continue
+			}
+		}
+		return true
+	}
+
+	return false
 }
 
 func (r *GitRepositoryReconciler) reconcileDelete(ctx context.Context, repository sourcev1.GitRepository) (ctrl.Result, error) {