diff --git a/pkg/app/piped/controller/planner.go b/pkg/app/piped/controller/planner.go index 76ef9a3692..cb046fd88b 100644 --- a/pkg/app/piped/controller/planner.go +++ b/pkg/app/piped/controller/planner.go @@ -159,6 +159,7 @@ func (p *planner) Run(ctx context.Context) error { in := pln.Input{ ApplicationID: p.deployment.ApplicationId, ApplicationName: p.deployment.ApplicationName, + PlatformProviderName: p.deployment.PlatformProvider, GitPath: *p.deployment.GitPath, Trigger: *p.deployment.Trigger, MostRecentSuccessfulCommitHash: p.lastSuccessfulCommitHash, diff --git a/pkg/app/piped/driftdetector/kubernetes/detector.go b/pkg/app/piped/driftdetector/kubernetes/detector.go index e5bee48e66..fe147ac9bc 100644 --- a/pkg/app/piped/driftdetector/kubernetes/detector.go +++ b/pkg/app/piped/driftdetector/kubernetes/detector.go @@ -23,6 +23,7 @@ import ( "time" "go.uber.org/zap" + "k8s.io/apimachinery/pkg/runtime/schema" "github.com/pipe-cd/pipecd/pkg/app/piped/livestatestore/kubernetes" provider "github.com/pipe-cd/pipecd/pkg/app/piped/platformprovider/kubernetes" @@ -121,6 +122,12 @@ func (d *detector) Run(ctx context.Context) error { } func (d *detector) check(ctx context.Context) { + isNamespacedResources, err := provider.GetIsNamespacedResources(d.provider.KubernetesConfig) + if err != nil { + d.logger.Error("failed to get isNamespacedResources", zap.Error(err)) + return + } + appsByRepo := d.listGroupedApplication() for repoID, apps := range appsByRepo { @@ -166,16 +173,16 @@ func (d *detector) check(ctx context.Context) { // Start checking all applications in this repository. for _, app := range apps { - if err := d.checkApplication(ctx, app, gitRepo, headCommit); err != nil { + if err := d.checkApplication(ctx, app, gitRepo, headCommit, isNamespacedResources); err != nil { d.logger.Error(fmt.Sprintf("failed to check application: %s", app.Id), zap.Error(err)) } } } } -func (d *detector) checkApplication(ctx context.Context, app *model.Application, repo git.Repo, headCommit git.Commit) error { +func (d *detector) checkApplication(ctx context.Context, app *model.Application, repo git.Repo, headCommit git.Commit, isNamespacedResources map[schema.GroupVersionKind]bool) error { watchingResourceKinds := d.stateGetter.GetWatchingResourceKinds() - headManifests, err := d.loadHeadManifests(ctx, app, repo, headCommit, watchingResourceKinds) + headManifests, err := d.loadHeadManifests(ctx, app, repo, headCommit, watchingResourceKinds, isNamespacedResources) if err != nil { return err } @@ -219,7 +226,7 @@ func (d *detector) checkApplication(ctx context.Context, app *model.Application, return d.reporter.ReportApplicationSyncState(ctx, app.Id, state) } -func (d *detector) loadHeadManifests(ctx context.Context, app *model.Application, repo git.Repo, headCommit git.Commit, watchingResourceKinds []provider.APIVersionKind) ([]provider.Manifest, error) { +func (d *detector) loadHeadManifests(ctx context.Context, app *model.Application, repo git.Repo, headCommit git.Commit, watchingResourceKinds []provider.APIVersionKind, isNamespacedResources map[schema.GroupVersionKind]bool) ([]provider.Manifest, error) { var ( manifestCache = provider.AppManifestsCache{ AppID: app.Id, @@ -281,7 +288,7 @@ func (d *detector) loadHeadManifests(ctx context.Context, app *model.Application } } - loader := provider.NewLoader(app.Name, appDir, repoDir, app.GitPath.ConfigFilename, cfg.KubernetesApplicationSpec.Input, d.gitClient, d.logger) + loader := provider.NewLoader(app.Name, appDir, repoDir, app.GitPath.ConfigFilename, cfg.KubernetesApplicationSpec.Input, isNamespacedResources, d.gitClient, d.logger) manifests, err = loader.LoadManifests(ctx) if err != nil { err = fmt.Errorf("failed to load new manifests: %w", err) diff --git a/pkg/app/piped/executor/kubernetes/baseline.go b/pkg/app/piped/executor/kubernetes/baseline.go index 1f9fb7b144..415a96755a 100644 --- a/pkg/app/piped/executor/kubernetes/baseline.go +++ b/pkg/app/piped/executor/kubernetes/baseline.go @@ -43,7 +43,24 @@ func (e *deployExecutor) ensureBaselineRollout(ctx context.Context) model.StageS // Load running manifests at the most successful deployed commit. e.LogPersister.Infof("Loading running manifests at commit %s for handling", runningCommit) - manifests, err := e.loadRunningManifests(ctx) + ds, err := e.RunningDSP.Get(ctx, e.LogPersister) + if err != nil { + e.LogPersister.Errorf("Failed to prepare running deploy source (%v)", err) + return model.StageStatus_STAGE_FAILURE + } + + loader := provider.NewLoader( + e.Deployment.ApplicationName, + ds.AppDir, + ds.RepoDir, + e.Deployment.GitPath.ConfigFilename, + e.appCfg.Input, + e.isNamespacedResources, + e.GitClient, + e.Logger, + ) + + manifests, err := loadManifests(ctx, e.Deployment.ApplicationId, runningCommit, e.AppManifestsCache, loader, e.Logger) if err != nil { e.LogPersister.Errorf("Failed while loading running manifests (%v)", err) return model.StageStatus_STAGE_FAILURE diff --git a/pkg/app/piped/executor/kubernetes/kubernetes.go b/pkg/app/piped/executor/kubernetes/kubernetes.go index 11436337c4..232d3a0bfb 100644 --- a/pkg/app/piped/executor/kubernetes/kubernetes.go +++ b/pkg/app/piped/executor/kubernetes/kubernetes.go @@ -24,6 +24,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "github.com/pipe-cd/pipecd/pkg/app/piped/executor" provider "github.com/pipe-cd/pipecd/pkg/app/piped/platformprovider/kubernetes" @@ -36,8 +37,9 @@ import ( type deployExecutor struct { executor.Input - commit string - appCfg *config.KubernetesApplicationSpec + commit string + appCfg *config.KubernetesApplicationSpec + isNamespacedResources map[schema.GroupVersionKind]bool loader provider.Loader applierGetter applierGetter @@ -75,6 +77,18 @@ func (e *deployExecutor) Execute(sig executor.StopSignal) model.StageStatus { ctx := sig.Context() e.commit = e.Deployment.Trigger.Commit.Hash + cp, ok := e.PipedConfig.FindPlatformProvider(e.Deployment.PlatformProvider, model.ApplicationKind_KUBERNETES) + if !ok { + e.LogPersister.Errorf("provider %s was not found", e.Deployment.PlatformProvider) + return model.StageStatus_STAGE_FAILURE + } + isNamespacedResources, err := provider.GetIsNamespacedResources(cp.KubernetesConfig) + if err != nil { + e.LogPersister.Errorf("failed to get isNamespacedResources %v", zap.Error(err)) + return model.StageStatus_STAGE_FAILURE + } + e.isNamespacedResources = isNamespacedResources + ds, err := e.TargetDSP.Get(ctx, e.LogPersister) if err != nil { e.LogPersister.Errorf("Failed to prepare target deploy source data (%v)", err) @@ -110,6 +124,7 @@ func (e *deployExecutor) Execute(sig executor.StopSignal) model.StageStatus { ds.RepoDir, e.Deployment.GitPath.ConfigFilename, e.appCfg.Input, + e.isNamespacedResources, e.GitClient, e.Logger, ) @@ -154,44 +169,7 @@ func (e *deployExecutor) Execute(sig executor.StopSignal) model.StageStatus { return executor.DetermineStageStatus(sig.Signal(), originalStatus, status) } -func (e *deployExecutor) loadRunningManifests(ctx context.Context) (manifests []provider.Manifest, err error) { - commit := e.Deployment.RunningCommitHash - if commit == "" { - return nil, fmt.Errorf("unable to determine running commit") - } - - loader := &manifestsLoadFunc{ - loadFunc: func(ctx context.Context) ([]provider.Manifest, error) { - ds, err := e.RunningDSP.Get(ctx, e.LogPersister) - if err != nil { - e.LogPersister.Errorf("Failed to prepare running deploy source (%v)", err) - return nil, err - } - - loader := provider.NewLoader( - e.Deployment.ApplicationName, - ds.AppDir, - ds.RepoDir, - e.Deployment.GitPath.ConfigFilename, - e.appCfg.Input, - e.GitClient, - e.Logger, - ) - return loader.LoadManifests(ctx) - }, - } - - return loadManifests(ctx, e.Deployment.ApplicationId, commit, e.AppManifestsCache, loader, e.Logger) -} - -type manifestsLoadFunc struct { - loadFunc func(context.Context) ([]provider.Manifest, error) -} - -func (l *manifestsLoadFunc) LoadManifests(ctx context.Context) ([]provider.Manifest, error) { - return l.loadFunc(ctx) -} - +// loadManifests loads the manifest using the given loader. It caches the loaded manifests for the given commit. func loadManifests(ctx context.Context, appID, commit string, manifestsCache cache.Cache, loader provider.Loader, logger *zap.Logger) (manifests []provider.Manifest, err error) { cache := provider.AppManifestsCache{ AppID: appID, diff --git a/pkg/app/piped/executor/kubernetes/primary.go b/pkg/app/piped/executor/kubernetes/primary.go index bdfcbe2b27..45328022fe 100644 --- a/pkg/app/piped/executor/kubernetes/primary.go +++ b/pkg/app/piped/executor/kubernetes/primary.go @@ -156,7 +156,26 @@ func (e *deployExecutor) ensurePrimaryRollout(ctx context.Context) model.StageSt // Find the running resources that are not defined in Git. e.LogPersister.Info("Start finding all running PRIMARY resources but no longer defined in Git") - runningManifests, err := e.loadRunningManifests(ctx) + // Load running manifests at the most successful deployed commit. + e.LogPersister.Infof("Loading running manifests at commit %s for handling", e.Deployment.RunningCommitHash) + ds, err := e.RunningDSP.Get(ctx, e.LogPersister) + if err != nil { + e.LogPersister.Errorf("Failed to prepare running deploy source (%v)", err) + return model.StageStatus_STAGE_FAILURE + } + + loader := provider.NewLoader( + e.Deployment.ApplicationName, + ds.AppDir, + ds.RepoDir, + e.Deployment.GitPath.ConfigFilename, + e.appCfg.Input, + e.isNamespacedResources, + e.GitClient, + e.Logger, + ) + + runningManifests, err := loadManifests(ctx, e.Deployment.ApplicationId, e.Deployment.RunningCommitHash, e.AppManifestsCache, loader, e.Logger) if err != nil { e.LogPersister.Errorf("Failed while loading running manifests (%v)", err) return model.StageStatus_STAGE_FAILURE diff --git a/pkg/app/piped/executor/kubernetes/rollback.go b/pkg/app/piped/executor/kubernetes/rollback.go index bccabfdb0e..8c10167fd0 100644 --- a/pkg/app/piped/executor/kubernetes/rollback.go +++ b/pkg/app/piped/executor/kubernetes/rollback.go @@ -22,6 +22,9 @@ import ( "strings" "go.uber.org/zap" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + "k8s.io/client-go/tools/clientcmd" "github.com/pipe-cd/pipecd/pkg/app/piped/executor" provider "github.com/pipe-cd/pipecd/pkg/app/piped/platformprovider/kubernetes" @@ -31,7 +34,8 @@ import ( type rollbackExecutor struct { executor.Input - appDir string + appDir string + isNamespacedResources map[schema.GroupVersionKind]bool } func (e *rollbackExecutor) Execute(sig executor.StopSignal) model.StageStatus { @@ -41,6 +45,39 @@ func (e *rollbackExecutor) Execute(sig executor.StopSignal) model.StageStatus { status model.StageStatus ) + // Use discovery to discover APIs supported by the Kubernetes API server. + // This should be run periodically with a low rate because the APIs are not added frequently. + // https://godoc.org/k8s.io/client-go/discovery + cp, ok := e.PipedConfig.FindPlatformProvider(e.Deployment.PlatformProvider, model.ApplicationKind_KUBERNETES) + if !ok { + e.LogPersister.Errorf("provider %s was not found", e.Deployment.PlatformProvider) + return model.StageStatus_STAGE_FAILURE + } + kubeConfig, err := clientcmd.BuildConfigFromFlags(cp.KubernetesConfig.MasterURL, cp.KubernetesConfig.KubeConfigPath) + if err != nil { + e.LogPersister.Errorf("failed to build kube config", zap.Error(err)) + return model.StageStatus_STAGE_FAILURE + } + discoveryClient, err := discovery.NewDiscoveryClientForConfig(kubeConfig) + if err != nil { + e.LogPersister.Errorf("failed to create discovery client: %v", zap.Error(err)) + return model.StageStatus_STAGE_FAILURE + } + groupResources, err := discoveryClient.ServerPreferredResources() + if err != nil { + e.LogPersister.Errorf("failed to fetch preferred resources: %v", zap.Error(err)) + return model.StageStatus_STAGE_FAILURE + } + e.LogPersister.Infof("successfully preferred resources that contains for %d groups", len(groupResources)) + + e.isNamespacedResources = make(map[schema.GroupVersionKind]bool) + for _, gr := range groupResources { + for _, resource := range gr.APIResources { + gvk := schema.FromAPIVersionAndKind(gr.GroupVersion, resource.Kind) + e.isNamespacedResources[gvk] = resource.Namespaced + } + } + switch model.Stage(e.Stage.Name) { case model.StageRollback: status = e.ensureRollback(ctx) @@ -82,7 +119,7 @@ func (e *rollbackExecutor) ensureRollback(ctx context.Context) model.StageStatus e.appDir = ds.AppDir - loader := provider.NewLoader(e.Deployment.ApplicationName, ds.AppDir, ds.RepoDir, e.Deployment.GitPath.ConfigFilename, appCfg.Input, e.GitClient, e.Logger) + loader := provider.NewLoader(e.Deployment.ApplicationName, ds.AppDir, ds.RepoDir, e.Deployment.GitPath.ConfigFilename, appCfg.Input, e.isNamespacedResources, e.GitClient, e.Logger) e.Logger.Info("start executing kubernetes stage", zap.String("stage-name", e.Stage.Name), zap.String("app-dir", ds.AppDir), diff --git a/pkg/app/piped/planner/kubernetes/kubernetes.go b/pkg/app/piped/planner/kubernetes/kubernetes.go index f2a5277936..ea1687ea87 100644 --- a/pkg/app/piped/planner/kubernetes/kubernetes.go +++ b/pkg/app/piped/planner/kubernetes/kubernetes.go @@ -24,6 +24,7 @@ import ( "time" "go.uber.org/zap" + "k8s.io/apimachinery/pkg/runtime/schema" "github.com/pipe-cd/pipecd/pkg/app/piped/deploysource" "github.com/pipe-cd/pipecd/pkg/app/piped/planner" @@ -40,6 +41,7 @@ const ( // Planner plans the deployment pipeline for kubernetes application. type Planner struct { + isNamespacedResources map[schema.GroupVersionKind]bool } type registerer interface { @@ -48,11 +50,25 @@ type registerer interface { // Register registers this planner into the given registerer. func Register(r registerer) { - r.Register(model.ApplicationKind_KUBERNETES, &Planner{}) + r.Register(model.ApplicationKind_KUBERNETES, &Planner{ + isNamespacedResources: make(map[schema.GroupVersionKind]bool), + }) } // Plan decides which pipeline should be used for the given input. func (p *Planner) Plan(ctx context.Context, in planner.Input) (out planner.Output, err error) { + cp, ok := in.PipedConfig.FindPlatformProvider(in.PlatformProviderName, model.ApplicationKind_KUBERNETES) + if !ok { + err = fmt.Errorf("provider %s was not found", in.PlatformProviderName) + return + } + isNamespacedResources, err := provider.GetIsNamespacedResources(cp.KubernetesConfig) + if err != nil { + err = fmt.Errorf("failed to get isNamespacedResources: %v", err) + return + } + p.isNamespacedResources = isNamespacedResources + ds, err := in.TargetDSP.Get(ctx, io.Discard) if err != nil { err = fmt.Errorf("error while preparing deploy source data (%v)", err) @@ -81,7 +97,7 @@ func (p *Planner) Plan(ctx context.Context, in planner.Input) (out planner.Outpu newManifests, ok := manifestCache.Get(in.Trigger.Commit.Hash) if !ok { // When the manifests were not in the cache we have to load them. - loader := provider.NewLoader(in.ApplicationName, ds.AppDir, ds.RepoDir, in.GitPath.ConfigFilename, cfg.Input, in.GitClient, in.Logger) + loader := provider.NewLoader(in.ApplicationName, ds.AppDir, ds.RepoDir, in.GitPath.ConfigFilename, cfg.Input, p.isNamespacedResources, in.GitClient, in.Logger) newManifests, err = loader.LoadManifests(ctx) if err != nil { return @@ -205,7 +221,7 @@ func (p *Planner) Plan(ctx context.Context, in planner.Input) (out planner.Outpu err = fmt.Errorf("unable to find the running configuration (%v)", err) return } - loader := provider.NewLoader(in.ApplicationName, runningDs.AppDir, runningDs.RepoDir, in.GitPath.ConfigFilename, runningCfg.Input, in.GitClient, in.Logger) + loader := provider.NewLoader(in.ApplicationName, runningDs.AppDir, runningDs.RepoDir, in.GitPath.ConfigFilename, runningCfg.Input, p.isNamespacedResources, in.GitClient, in.Logger) oldManifests, err = loader.LoadManifests(ctx) if err != nil { err = fmt.Errorf("failed to load previously deployed manifests: %w", err) diff --git a/pkg/app/piped/planner/planner.go b/pkg/app/piped/planner/planner.go index dccca4bb53..cba5cdc6ed 100644 --- a/pkg/app/piped/planner/planner.go +++ b/pkg/app/piped/planner/planner.go @@ -44,6 +44,7 @@ type gitClient interface { type Input struct { ApplicationID string ApplicationName string + PlatformProviderName string GitPath model.ApplicationGitPath Trigger model.DeploymentTrigger MostRecentSuccessfulCommitHash string diff --git a/pkg/app/piped/planpreview/builder.go b/pkg/app/piped/planpreview/builder.go index 315e8f30a6..e6cb1962fb 100644 --- a/pkg/app/piped/planpreview/builder.go +++ b/pkg/app/piped/planpreview/builder.go @@ -335,9 +335,10 @@ func (b *builder) plan(ctx context.Context, app *model.Application, targetDSP de } in := planner.Input{ - ApplicationID: app.Id, - ApplicationName: app.Name, - GitPath: *app.GitPath, + ApplicationID: app.Id, + ApplicationName: app.Name, + PlatformProviderName: app.PlatformProvider, + GitPath: *app.GitPath, Trigger: model.DeploymentTrigger{ Commit: &model.Commit{ Branch: b.repoCfg.Branch, diff --git a/pkg/app/piped/planpreview/kubernetesdiff.go b/pkg/app/piped/planpreview/kubernetesdiff.go index 86d5ae8872..ba082dad45 100644 --- a/pkg/app/piped/planpreview/kubernetesdiff.go +++ b/pkg/app/piped/planpreview/kubernetesdiff.go @@ -21,6 +21,9 @@ import ( "io" "go.uber.org/zap" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + "k8s.io/client-go/tools/clientcmd" "github.com/pipe-cd/pipecd/pkg/app/piped/deploysource" provider "github.com/pipe-cd/pipecd/pkg/app/piped/platformprovider/kubernetes" @@ -40,7 +43,39 @@ func (b *builder) kubernetesDiff( var oldManifests, newManifests []provider.Manifest var err error - newManifests, err = loadKubernetesManifests(ctx, *app, targetDSP, b.appManifestsCache, b.gitClient, b.logger) + // Use discovery to discover APIs supported by the Kubernetes API server. + // This should be run periodically with a low rate because the APIs are not added frequently. + // https://godoc.org/k8s.io/client-go/discovery + cp, ok := b.pipedCfg.FindPlatformProvider(app.PlatformProvider, model.ApplicationKind_KUBERNETES) + if !ok { + err = fmt.Errorf("provider %s was not found", app.PlatformProvider) + return nil, err + } + kubeConfig, err := clientcmd.BuildConfigFromFlags(cp.KubernetesConfig.MasterURL, cp.KubernetesConfig.KubeConfigPath) + if err != nil { + err = fmt.Errorf("failed to build kube config: %w", err) + return nil, err + } + discoveryClient, err := discovery.NewDiscoveryClientForConfig(kubeConfig) + if err != nil { + err = fmt.Errorf("failed to create discovery client: %w", err) + return nil, err + } + groupResources, err := discoveryClient.ServerPreferredResources() + if err != nil { + err = fmt.Errorf("failed to fetch preferred resources: %w", err) + return nil, err + } + + isNamespacedResources := make(map[schema.GroupVersionKind]bool) + for _, gr := range groupResources { + for _, resource := range gr.APIResources { + gvk := schema.FromAPIVersionAndKind(gr.GroupVersion, resource.Kind) + isNamespacedResources[gvk] = resource.Namespaced + } + } + + newManifests, err = loadKubernetesManifests(ctx, *app, targetDSP, b.appManifestsCache, isNamespacedResources, b.gitClient, b.logger) if err != nil { fmt.Fprintf(buf, "failed to load kubernetes manifests at the head commit (%v)\n", err) return nil, err @@ -53,7 +88,7 @@ func (b *builder) kubernetesDiff( *app.GitPath, b.secretDecrypter, ) - oldManifests, err = loadKubernetesManifests(ctx, *app, runningDSP, b.appManifestsCache, b.gitClient, b.logger) + oldManifests, err = loadKubernetesManifests(ctx, *app, runningDSP, b.appManifestsCache, isNamespacedResources, b.gitClient, b.logger) if err != nil { fmt.Fprintf(buf, "failed to load kubernetes manifests at the running commit (%v)\n", err) return nil, err @@ -92,7 +127,7 @@ func (b *builder) kubernetesDiff( }, nil } -func loadKubernetesManifests(ctx context.Context, app model.Application, dsp deploysource.Provider, manifestsCache cache.Cache, gc gitClient, logger *zap.Logger) (manifests []provider.Manifest, err error) { +func loadKubernetesManifests(ctx context.Context, app model.Application, dsp deploysource.Provider, manifestsCache cache.Cache, isNamespacedResources map[schema.GroupVersionKind]bool, gc gitClient, logger *zap.Logger) (manifests []provider.Manifest, err error) { commit := dsp.Revision() cache := provider.AppManifestsCache{ AppID: app.Id, @@ -122,6 +157,7 @@ func loadKubernetesManifests(ctx context.Context, app model.Application, dsp dep ds.RepoDir, app.GitPath.ConfigFilename, appCfg.Input, + isNamespacedResources, gc, logger, ) diff --git a/pkg/app/piped/platformprovider/kubernetes/applier.go b/pkg/app/piped/platformprovider/kubernetes/applier.go index e3c416da89..47d69bb5bf 100644 --- a/pkg/app/piped/platformprovider/kubernetes/applier.go +++ b/pkg/app/piped/platformprovider/kubernetes/applier.go @@ -158,7 +158,7 @@ func (a *applier) Delete(ctx context.Context, k ResourceKey) (err error) { return err } - if k.String() != m.GetAnnotations()[LabelResourceKey] { + if k.String() != m.Key.String() { return ErrNotFound } diff --git a/pkg/app/piped/platformprovider/kubernetes/kubernetes.go b/pkg/app/piped/platformprovider/kubernetes/kubernetes.go index 46995a2500..e47b2f3af9 100644 --- a/pkg/app/piped/platformprovider/kubernetes/kubernetes.go +++ b/pkg/app/piped/platformprovider/kubernetes/kubernetes.go @@ -16,6 +16,12 @@ package kubernetes import ( "errors" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + "k8s.io/client-go/tools/clientcmd" + + "github.com/pipe-cd/pipecd/pkg/config" ) var ( @@ -42,3 +48,31 @@ const ( kustomizationFileName = "kustomization.yaml" ) + +// GetIsNamespacedResources return the map to determine whether the given GroupVersionKind is namespaced or not. +// The key is GroupVersionKind and the value is a boolean value. +// This function will get the information from the Kubernetes cluster using the given PlatformProviderKubernetesConfig. +func GetIsNamespacedResources(cp *config.PlatformProviderKubernetesConfig) (map[schema.GroupVersionKind]bool, error) { + kubeConfig, err := clientcmd.BuildConfigFromFlags(cp.MasterURL, cp.KubeConfigPath) + if err != nil { + return nil, err + } + discoveryClient, err := discovery.NewDiscoveryClientForConfig(kubeConfig) + if err != nil { + return nil, err + } + groupResources, err := discoveryClient.ServerPreferredResources() + if err != nil { + return nil, err + } + + isNamespacedResources := make(map[schema.GroupVersionKind]bool) + for _, gr := range groupResources { + for _, resource := range gr.APIResources { + gvk := schema.FromAPIVersionAndKind(gr.GroupVersion, resource.Kind) + isNamespacedResources[gvk] = resource.Namespaced + } + } + + return isNamespacedResources, nil +} diff --git a/pkg/app/piped/platformprovider/kubernetes/loader.go b/pkg/app/piped/platformprovider/kubernetes/loader.go index 9f2eb8ca62..52be941404 100644 --- a/pkg/app/piped/platformprovider/kubernetes/loader.go +++ b/pkg/app/piped/platformprovider/kubernetes/loader.go @@ -25,6 +25,7 @@ import ( "sync" "go.uber.org/zap" + "k8s.io/apimachinery/pkg/runtime/schema" "github.com/pipe-cd/pipecd/pkg/app/piped/toolregistry" "github.com/pipe-cd/pipecd/pkg/config" @@ -49,13 +50,14 @@ type gitClient interface { } type loader struct { - appName string - appDir string - repoDir string - configFileName string - input config.KubernetesDeploymentInput - gc gitClient - logger *zap.Logger + appName string + appDir string + repoDir string + configFileName string + input config.KubernetesDeploymentInput + isNamespacedResources map[schema.GroupVersionKind]bool + gc gitClient + logger *zap.Logger templatingMethod TemplatingMethod kustomize *Kustomize @@ -67,27 +69,26 @@ type loader struct { func NewLoader( appName, appDir, repoDir, configFileName string, input config.KubernetesDeploymentInput, + isNamespacedResources map[schema.GroupVersionKind]bool, gc gitClient, logger *zap.Logger, ) Loader { return &loader{ - appName: appName, - appDir: appDir, - repoDir: repoDir, - configFileName: configFileName, - input: input, - gc: gc, - logger: logger.Named("kubernetes-loader"), + appName: appName, + appDir: appDir, + repoDir: repoDir, + configFileName: configFileName, + input: input, + isNamespacedResources: isNamespacedResources, + gc: gc, + logger: logger.Named("kubernetes-loader"), } } // LoadManifests renders and loads all manifests for application. func (l *loader) LoadManifests(ctx context.Context) (manifests []Manifest, err error) { defer func() { - // Override namespace if set because ParseManifests does not parse it - // if namespace is not explicitly specified in the manifests. - setNamespace(manifests, l.input.Namespace) sortManifests(manifests) }() l.initOnce.Do(func() { @@ -166,16 +167,50 @@ func (l *loader) LoadManifests(ctx context.Context) (manifests []Manifest, err e err = fmt.Errorf("unsupport templating method %v", l.templatingMethod) } + // Refine the namespace when reading manifests from git repo + // because the namespace is determined not only by its own namespace on the file but also the namespace on the app.pipecd.yaml. + for i := range manifests { + m := manifests[i] + err := l.determineNamespace(&m) + if err != nil { + return nil, err + } + } + return } -func setNamespace(manifests []Manifest, namespace string) { - if namespace == "" { - return +// determineNamespace fix the namespace of the given manifest. +// The priority is as follows: +// If the resource is cluster-scoped, it returns an empty string. +// Otherwise, it is the namespace-scoped resource and the namespace is determined by the following order: +// 1. The namespace set in the application configuration. +// 2. The namespace set in the manifest. +// 3. The default namespace. +func (l *loader) determineNamespace(m *Manifest) error { + namespaced, ok := l.isNamespacedResources[m.u.GroupVersionKind()] + if !ok { + return fmt.Errorf("unknown resource kind %s", m.u.GroupVersionKind().String()) } - for i := range manifests { - manifests[i].Key.Namespace = namespace + + namespace := "" // empty if cluster-scoped resource + + if namespaced { + namespace = "default" + + if ns := m.u.GetNamespace(); ns != "" { + namespace = ns + } + + if l.input.Namespace != "" { + namespace = l.input.Namespace + } } + + m.Key.Namespace = namespace + m.u.SetNamespace(namespace) + + return nil } func sortManifests(manifests []Manifest) { diff --git a/pkg/app/piped/platformprovider/kubernetes/loader_test.go b/pkg/app/piped/platformprovider/kubernetes/loader_test.go index c553206729..1a7281324a 100644 --- a/pkg/app/piped/platformprovider/kubernetes/loader_test.go +++ b/pkg/app/piped/platformprovider/kubernetes/loader_test.go @@ -19,6 +19,9 @@ import ( "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/pipe-cd/pipecd/pkg/config" ) func TestSortManifests(t *testing.T) { @@ -76,3 +79,139 @@ func TestSortManifests(t *testing.T) { }) } } + +func Test_loader_determineNamespace(t *testing.T) { + testcases := []struct { + name string + manifest Manifest + isNamespacedResources map[schema.GroupVersionKind]bool + cfgK8sInput config.KubernetesDeploymentInput + want string + wantErr bool + }{ + { + name: "failed because unknown resource kind", + manifest: Manifest{ + u: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "unknown", + "kind": "Unknown", + }, + }, + }, + isNamespacedResources: map[schema.GroupVersionKind]bool{}, + want: "", + wantErr: true, + }, + { + name: "cluster-scoped resource: use '' as namespace", + manifest: Manifest{ + u: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Namespace", + }, + }, + }, + isNamespacedResources: map[schema.GroupVersionKind]bool{ + {Group: "", Version: "v1", Kind: "Namespace"}: false, + }, + want: "", + wantErr: false, + }, + { + name: "cluster-scoped resource: use '' even though the app.pipecd.yaml has 'spec.input.namespace'", + manifest: Manifest{ + u: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Namespace", + }, + }, + }, + isNamespacedResources: map[schema.GroupVersionKind]bool{ + {Group: "", Version: "v1", Kind: "Namespace"}: false, + }, + cfgK8sInput: config.KubernetesDeploymentInput{ + Namespace: "test", + }, + want: "", + wantErr: false, + }, + { + name: "namespace-scoped resource: use the namespace set in the app.pipecd.yaml if it is not empty and the manifest has no namespace", + manifest: Manifest{ + u: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": map[string]interface{}{}, + }, + }, + }, + isNamespacedResources: map[schema.GroupVersionKind]bool{ + {Group: "", Version: "v1", Kind: "Pod"}: true, + }, + cfgK8sInput: config.KubernetesDeploymentInput{ + Namespace: "inputNamespace", + }, + want: "inputNamespace", + wantErr: false, + }, + { + name: "namespace-scoped resource: use the namespace set in the app.pipecd.yaml even though the manifest has a namespace", + manifest: Manifest{ + u: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": map[string]interface{}{ + "namespace": "test", + }, + }, + }, + }, + isNamespacedResources: map[schema.GroupVersionKind]bool{ + {Group: "", Version: "v1", Kind: "Pod"}: true, + }, + cfgK8sInput: config.KubernetesDeploymentInput{ + Namespace: "inputNamespace", + }, + want: "inputNamespace", + wantErr: false, + }, + { + name: "namespace-scoped resource: use 'default' namespace when input namespace is empty and the namespace in the app.pipecd.yaml is empty", + manifest: Manifest{ + u: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": map[string]interface{}{}, + }, + }, + }, + isNamespacedResources: map[schema.GroupVersionKind]bool{ + {Group: "", Version: "v1", Kind: "Pod"}: true, + }, + cfgK8sInput: config.KubernetesDeploymentInput{}, + want: "default", + wantErr: false, + }, + } + + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + l := &loader{ + isNamespacedResources: tc.isNamespacedResources, + input: tc.cfgK8sInput, + } + err := l.determineNamespace(&tc.manifest) + + assert.Equal(t, tc.wantErr, err != nil) + assert.Equal(t, tc.want, tc.manifest.Key.Namespace) + assert.Equal(t, tc.want, tc.manifest.u.GetNamespace()) + }) + } +} diff --git a/pkg/app/piped/platformprovider/kubernetes/manifest_test.go b/pkg/app/piped/platformprovider/kubernetes/manifest_test.go index 947f027f82..18038175e0 100644 --- a/pkg/app/piped/platformprovider/kubernetes/manifest_test.go +++ b/pkg/app/piped/platformprovider/kubernetes/manifest_test.go @@ -24,12 +24,13 @@ import ( func TestParseManifests(t *testing.T) { maker := func(name, kind string, metadata map[string]interface{}) Manifest { + namespace, _ := metadata["namespace"].(string) return Manifest{ Key: ResourceKey{ APIVersion: "v1", Kind: kind, Name: name, - Namespace: "default", + Namespace: namespace, }, u: &unstructured.Unstructured{ Object: map[string]interface{}{ @@ -72,11 +73,13 @@ apiVersion: v1 kind: ConfigMap metadata: name: envoy-config + namespace: default creationTimestamp: "2022-12-09T01:23:45Z" `, want: []Manifest{ maker("envoy-config", "ConfigMap", map[string]interface{}{ "name": "envoy-config", + "namespace": "default", "creationTimestamp": "2022-12-09T01:23:45Z", }), }, @@ -88,13 +91,15 @@ apiVersion: v1 kind: Kind1 metadata: name: config + namespace: default extra: | single-new-line `, want: []Manifest{ maker("config", "Kind1", map[string]interface{}{ - "name": "config", - "extra": "single-new-line\n", + "name": "config", + "namespace": "default", + "extra": "single-new-line\n", }), }, }, @@ -105,12 +110,14 @@ apiVersion: v1 kind: Kind1 metadata: name: config + namespace: default extra: | no-new-line`, want: []Manifest{ maker("config", "Kind1", map[string]interface{}{ - "name": "config", - "extra": "no-new-line", + "name": "config", + "namespace": "default", + "extra": "no-new-line", }), }, }, @@ -121,6 +128,7 @@ apiVersion: v1 kind: Kind1 metadata: name: config1 + namespace: default extra: |- no-new-line --- @@ -128,6 +136,7 @@ apiVersion: v1 kind: Kind2 metadata: name: config2 + namespace: default extra: | single-new-line-1 --- @@ -135,6 +144,7 @@ apiVersion: v1 kind: Kind3 metadata: name: config3 + namespace: default extra: | single-new-line-2 @@ -144,6 +154,7 @@ apiVersion: v1 kind: Kind4 metadata: name: config4 + namespace: default extra: |+ multiple-new-line-1 @@ -153,6 +164,7 @@ apiVersion: v1 kind: Kind5 metadata: name: config5 + namespace: default extra: |+ multiple-new-line-2 @@ -160,24 +172,29 @@ metadata: `, want: []Manifest{ maker("config1", "Kind1", map[string]interface{}{ - "name": "config1", - "extra": "no-new-line", + "name": "config1", + "namespace": "default", + "extra": "no-new-line", }), maker("config2", "Kind2", map[string]interface{}{ - "name": "config2", - "extra": "single-new-line-1\n", + "name": "config2", + "namespace": "default", + "extra": "single-new-line-1\n", }), maker("config3", "Kind3", map[string]interface{}{ - "name": "config3", - "extra": "single-new-line-2\n", + "name": "config3", + "namespace": "default", + "extra": "single-new-line-2\n", }), maker("config4", "Kind4", map[string]interface{}{ - "name": "config4", - "extra": "multiple-new-line-1\n\n\n", + "name": "config4", + "namespace": "default", + "extra": "multiple-new-line-1\n\n\n", }), maker("config5", "Kind5", map[string]interface{}{ - "name": "config5", - "extra": "multiple-new-line-2\n\n\n", + "name": "config5", + "namespace": "default", + "extra": "multiple-new-line-2\n\n\n", }), }, }, diff --git a/pkg/app/piped/platformprovider/kubernetes/resourcekey.go b/pkg/app/piped/platformprovider/kubernetes/resourcekey.go index 7467e9d133..0ea3ff16a0 100644 --- a/pkg/app/piped/platformprovider/kubernetes/resourcekey.go +++ b/pkg/app/piped/platformprovider/kubernetes/resourcekey.go @@ -253,9 +253,6 @@ func MakeResourceKey(obj *unstructured.Unstructured) ResourceKey { Namespace: obj.GetNamespace(), Name: obj.GetName(), } - if k.Namespace == "" { - k.Namespace = DefaultNamespace - } return k }