diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 542ee1fd..d2a72e69 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -74,6 +74,21 @@ jobs: kubectl -n impersonation wait kustomizations/podinfo --for=condition=ready --timeout=4m kubectl -n impersonation delete kustomizations/podinfo until kubectl -n impersonation get deploy/podinfo 2>&1 | grep NotFound ; do sleep 2; done + - name: Run user impersonation tests + run: | + kubectl -n kustomize-system apply -f ./config/testdata/user-impersonation/new-rbac.yaml + kubectl -n kustomize-system patch deploy/kustomize-controller -p "$(cat ./config/testdata/user-impersonation/patch.yaml)" + kubectl -n tenant-a apply -f ./config/testdata/user-impersonation/use-test.yaml + kubectl -n tenant-a wait kustomizations/podinfo --for=condition=ready --timeout=4m + kubectl -n tenant-a delete kustomizations/podinfo --wait + kubectl -n tenant-b apply -f ./config/testdata/user-impersonation/sa-test.yaml + kubectl -n tenant-b wait kustomizations/podinfo --for=condition=ready --timeout=4m + kubectl -n tenant-b delete kustomizations/podinfo --wait + kubectl -n tenant-d apply -f ./config/testdata/user-impersonation/token-imp.yaml + kubectl -n tenant-d wait kustomizations/podinfo --for=condition=ready --timeout=4m + kubectl -n tenant-d delete kustomizations/podinfo --wait + kubectl -n tenant-c apply -f ./config/testdata/user-impersonation/fail-sa-test.yaml + until kubectl -n tenant-c get kustomization podinfo -oyaml | grep "Error from server (Forbidden)" ; do sleep 2; done - name: Logs run: | kubectl -n kustomize-system logs deploy/source-controller diff --git a/Makefile b/Makefile index 2bacc911..b63b56a5 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,10 @@ manager: generate fmt vet run: generate fmt vet manifests go run ./main.go --metrics-addr=:8089 +# Run against the configured Kubernetes cluster in ~/.kube/config with flux user enabled +run-enable-user: generate fmt vet manifests + go run ./main.go --metrics-addr=:8089 --user-impersonation + # Download the CRDs the controller depends on download-crd-deps: curl -s https://raw.githubusercontent.com/fluxcd/source-controller/${SOURCE_VER}/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml > config/crd/bases/gitrepositories.yaml diff --git a/api/v1beta1/kustomization_types.go b/api/v1beta1/kustomization_types.go index f2c0c8ec..1affc0ef 100644 --- a/api/v1beta1/kustomization_types.go +++ b/api/v1beta1/kustomization_types.go @@ -138,6 +138,21 @@ type KustomizationSpec struct { // +kubebuilder:default:=false // +optional Force bool `json:"force,omitempty"` + + // Principal provides details on how the controller should + // carry out impersonation during garbage collections, health-check + // and applies + Principal *Principal `json:"principal,omitempty"` +} + +type Principal struct { + // Kind specifies the kind of object to be impersonated + // The kind could be 'User' or 'ServiceAccount' + // +kubebuilder:validation:Enum=ServiceAccount;User + Kind string `json:"kind"` + + // The name of the object to be impersonated + Name string `json:"name"` } // Decryption defines how decryption is handled for Kubernetes manifests. diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 74448f61..9716746c 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -198,6 +198,11 @@ func (in *KustomizationSpec) DeepCopyInto(out *KustomizationSpec) { *out = new(v1.Duration) **out = **in } + if in.Principal != nil { + in, out := &in.Principal, &out.Principal + *out = new(Principal) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KustomizationSpec. @@ -265,6 +270,21 @@ func (in *PostBuild) DeepCopy() *PostBuild { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Principal) DeepCopyInto(out *Principal) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Principal. +func (in *Principal) DeepCopy() *Principal { + if in == nil { + return nil + } + out := new(Principal) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Snapshot) DeepCopyInto(out *Snapshot) { *out = *in diff --git a/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml b/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml index ce2614a0..9391ea5e 100644 --- a/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml +++ b/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml @@ -297,6 +297,25 @@ spec: type: object type: array type: object + principal: + description: Principal provides details on how the controller should + carry out impersonation during garbage collections, health-check + and applies + properties: + kind: + description: Kind specifies the kind of object to be impersonated + The kind could be 'User' or 'ServiceAccount' + enum: + - ServiceAccount + - User + type: string + name: + description: The name of the object to be impersonated + type: string + required: + - kind + - name + type: object prune: description: Prune enables garbage collection. type: boolean diff --git a/config/testdata/user-impersonation/fail-sa-test.yaml b/config/testdata/user-impersonation/fail-sa-test.yaml new file mode 100644 index 00000000..4effd164 --- /dev/null +++ b/config/testdata/user-impersonation/fail-sa-test.yaml @@ -0,0 +1,84 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: tenant-c +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tenant-c + namespace: tenant-c +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: tenant-c-admin + namespace: tenant-c +rules: + - apiGroups: ['apps'] + resources: ['deployments'] + verbs: ['*'] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: tenant-c-admin + namespace: tenant-c +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: tenant-c-admin +subjects: + - kind: ServiceAccount + name: tenant-c + namespace: tenant-c +--- +# permissions for flux:users groups shouldn't work with serviceaccount +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: flux-users + namespace: tenant-c +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: Group + name: flux:users + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: source.toolkit.fluxcd.io/v1beta1 +kind: GitRepository +metadata: + name: podinfo + namespace: tenant-c +spec: + interval: 5m + url: https://github.com/stefanprodan/podinfo + ref: + tag: "5.0.3" +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1beta1 +kind: Kustomization +metadata: + name: podinfo + namespace: tenant-c +spec: + targetNamespace: tenant-c + principal: + kind: ServiceAccount + name: tenant-c + interval: 5m + path: "./kustomize" + prune: true + sourceRef: + kind: GitRepository + name: podinfo + namespace: tenant-c + validation: client + timeout: 2m + healthChecks: + - kind: Deployment + name: podinfo + namespace: tenant-c diff --git a/config/testdata/user-impersonation/new-rbac.yaml b/config/testdata/user-impersonation/new-rbac.yaml new file mode 100644 index 00000000..11de7785 --- /dev/null +++ b/config/testdata/user-impersonation/new-rbac.yaml @@ -0,0 +1,135 @@ + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: user-impersonation + namespace: kustomize-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: user-impersonation-role +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - "" + resources: + - configmaps/status + verbs: + - get + - update + - patch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - "" + resources: + - secrets + - serviceaccounts + verbs: + - get + - list + - watch + - apiGroups: + - kustomize.toolkit.fluxcd.io + resources: + - kustomizations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - kustomize.toolkit.fluxcd.io + resources: + - kustomizations/finalizers + verbs: + - create + - delete + - get + - patch + - update + - apiGroups: + - kustomize.toolkit.fluxcd.io + resources: + - kustomizations/status + verbs: + - get + - patch + - update + - apiGroups: + - source.toolkit.fluxcd.io + resources: + - buckets + - gitrepositories + verbs: + - get + - list + - watch + - apiGroups: + - source.toolkit.fluxcd.io + resources: + - buckets/status + - gitrepositories/status + verbs: + - get + - apiGroups: + - "" + resources: + - groups + - users + - serviceaccounts + verbs: + - impersonate +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: user-impersonation +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: user-impersonation-role +subjects: + - kind: ServiceAccount + name: user-impersonation + namespace: kustomize-system diff --git a/config/testdata/user-impersonation/patch.yaml b/config/testdata/user-impersonation/patch.yaml new file mode 100644 index 00000000..a1593eed --- /dev/null +++ b/config/testdata/user-impersonation/patch.yaml @@ -0,0 +1,12 @@ +spec: + template: + spec: + containers: + - name: manager + args: + - --watch-all-namespaces=true + - --log-level=info + - --log-encoding=json + - --enable-leader-election + - --user-impersonation + serviceAccountName: user-impersonation diff --git a/config/testdata/user-impersonation/sa-test.yaml b/config/testdata/user-impersonation/sa-test.yaml new file mode 100644 index 00000000..79cad36b --- /dev/null +++ b/config/testdata/user-impersonation/sa-test.yaml @@ -0,0 +1,69 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: tenant-b +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tenant-b + namespace: tenant-b +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: tenant-b-admin + namespace: tenant-b +rules: + - apiGroups: ['*'] + resources: ['*'] + verbs: ['*'] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: tenant-b-admin + namespace: tenant-b +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: tenant-b-admin +subjects: + - kind: ServiceAccount + name: tenant-b + namespace: tenant-b +--- +apiVersion: source.toolkit.fluxcd.io/v1beta1 +kind: GitRepository +metadata: + name: podinfo + namespace: tenant-b +spec: + interval: 5m + url: https://github.com/stefanprodan/podinfo + ref: + tag: "5.0.3" +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1beta1 +kind: Kustomization +metadata: + name: podinfo + namespace: tenant-b +spec: + targetNamespace: tenant-b + principal: + kind: ServiceAccount + name: tenant-b + interval: 5m + path: "./kustomize" + prune: true + sourceRef: + kind: GitRepository + name: podinfo + namespace: tenant-b + validation: client + timeout: 2m + healthChecks: + - kind: Deployment + name: podinfo + namespace: tenant-b diff --git a/config/testdata/user-impersonation/token-imp.yaml b/config/testdata/user-impersonation/token-imp.yaml new file mode 100644 index 00000000..c6714c4f --- /dev/null +++ b/config/testdata/user-impersonation/token-imp.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: tenant-d +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tenant-d + namespace: tenant-d +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: tenant-d-admin + namespace: tenant-d +rules: + - apiGroups: ['*'] + resources: ['*'] + verbs: ['*'] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: tenant-d-admin + namespace: tenant-d +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: tenant-d-admin +subjects: + - kind: ServiceAccount + name: tenant-d + namespace: tenant-d +--- +apiVersion: source.toolkit.fluxcd.io/v1beta1 +kind: GitRepository +metadata: + name: podinfo + namespace: tenant-d +spec: + interval: 5m + url: https://github.com/stefanprodan/podinfo + ref: + tag: "5.0.3" +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1beta1 +kind: Kustomization +metadata: + name: podinfo + namespace: tenant-d +spec: + targetNamespace: tenant-d + # reconciler should use token impersonation when principal is nil ans serviceAccountName is specified + serviceAccountName: tenant-d + interval: 5m + path: "./kustomize" + prune: true + sourceRef: + kind: GitRepository + name: podinfo + namespace: tenant-d + validation: client + timeout: 2m + healthChecks: + - kind: Deployment + name: podinfo + namespace: tenant-d diff --git a/config/testdata/user-impersonation/use-test.yaml b/config/testdata/user-impersonation/use-test.yaml new file mode 100644 index 00000000..9bad188c --- /dev/null +++ b/config/testdata/user-impersonation/use-test.yaml @@ -0,0 +1,63 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: tenant-a +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: tenant-a-admin + namespace: tenant-a +rules: + - apiGroups: ['*'] + resources: ['*'] + verbs: ['*'] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: tenant-a-admin + namespace: tenant-a +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: tenant-a-admin +subjects: + - kind: User + name: flux:user:tenant-a:tenant-a + namespace: tenant-a +--- +apiVersion: source.toolkit.fluxcd.io/v1beta1 +kind: GitRepository +metadata: + name: podinfo + namespace: tenant-a +spec: + interval: 5m + url: https://github.com/stefanprodan/podinfo + ref: + tag: "5.0.3" +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1beta1 +kind: Kustomization +metadata: + name: podinfo + namespace: tenant-a +spec: + targetNamespace: tenant-a + principal: + kind: User + name: tenant-a + interval: 5m + path: "./kustomize" + prune: true + sourceRef: + kind: GitRepository + name: podinfo + namespace: tenant-a + validation: client + timeout: 2m + healthChecks: + - kind: Deployment + name: podinfo + namespace: tenant-a diff --git a/controllers/kustomization_controller.go b/controllers/kustomization_controller.go index 3a05106e..04f7c77d 100644 --- a/controllers/kustomization_controller.go +++ b/controllers/kustomization_controller.go @@ -30,12 +30,7 @@ import ( "time" securejoin "github.com/cyphar/filepath-securejoin" - "github.com/fluxcd/pkg/apis/meta" - "github.com/fluxcd/pkg/runtime/events" - "github.com/fluxcd/pkg/runtime/metrics" - "github.com/fluxcd/pkg/runtime/predicates" - "github.com/fluxcd/pkg/untar" - sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" + pkgclient "github.com/fluxcd/pkg/runtime/client" "github.com/go-logr/logr" "github.com/hashicorp/go-retryablehttp" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -43,6 +38,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" kuberecorder "k8s.io/client-go/tools/record" "k8s.io/client-go/tools/reference" "sigs.k8s.io/cli-utils/pkg/kstatus/polling" @@ -56,6 +52,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" "sigs.k8s.io/kustomize/api/filesys" + "github.com/fluxcd/pkg/apis/meta" + "github.com/fluxcd/pkg/runtime/events" + "github.com/fluxcd/pkg/runtime/metrics" + "github.com/fluxcd/pkg/runtime/predicates" + "github.com/fluxcd/pkg/untar" + sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" + kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1" ) @@ -70,13 +73,14 @@ import ( // KustomizationReconciler reconciles a Kustomization object type KustomizationReconciler struct { client.Client - httpClient *retryablehttp.Client - requeueDependency time.Duration - Scheme *runtime.Scheme - EventRecorder kuberecorder.EventRecorder - ExternalEventRecorder *events.Recorder - MetricsRecorder *metrics.Recorder - StatusPoller *polling.StatusPoller + httpClient *retryablehttp.Client + requeueDependency time.Duration + Scheme *runtime.Scheme + EventRecorder kuberecorder.EventRecorder + ExternalEventRecorder *events.Recorder + MetricsRecorder *metrics.Recorder + StatusPoller *polling.StatusPoller + EnableUserImpersonation bool } type KustomizationReconcilerOptions struct { @@ -315,7 +319,7 @@ func (r *KustomizationReconciler) reconcile( } // create any necessary kube-clients for impersonation - impersonation := NewKustomizeImpersonation(kustomization, r.Client, r.StatusPoller, dirPath) + impersonation := NewKustomizeImpersonation(kustomization, r.EnableUserImpersonation, r.Client, r.StatusPoller, dirPath) kubeClient, statusPoller, err := impersonation.GetClient(ctx) if err != nil { return kustomizev1.KustomizationNotReady( @@ -590,22 +594,9 @@ func (r *KustomizationReconciler) validate(ctx context.Context, kustomization ku cmd := fmt.Sprintf("cd %s && kubectl apply -f %s.yaml --timeout=%s --dry-run=%s --cache-dir=/tmp --force=%t", dirPath, kustomization.GetUID(), kustomization.GetTimeout().String(), validation, kustomization.Spec.Force) - if kustomization.Spec.KubeConfig != nil { - kubeConfig, err := imp.WriteKubeConfig(ctx) - if err != nil { - return err - } - cmd = fmt.Sprintf("%s --kubeconfig=%s", cmd, kubeConfig) - } else { - // impersonate SA - if kustomization.Spec.ServiceAccountName != "" { - saToken, err := imp.GetServiceAccountToken(ctx) - if err != nil { - return fmt.Errorf("service account impersonation failed: %w", err) - } - - cmd = fmt.Sprintf("%s --token %s", cmd, saToken) - } + cmd, err := r.buildKubectlCmdImpersonation(ctx, kustomization, imp, cmd) + if err != nil { + return err } command := exec.CommandContext(applyCtx, "/bin/sh", "-c", cmd) @@ -630,22 +621,9 @@ func (r *KustomizationReconciler) apply(ctx context.Context, kustomization kusto cmd := fmt.Sprintf("cd %s && kubectl apply --field-manager=%s -f %s.yaml --timeout=%s --cache-dir=/tmp --force=%t", dirPath, fieldManager, kustomization.GetUID(), kustomization.Spec.Interval.Duration.String(), kustomization.Spec.Force) - if kustomization.Spec.KubeConfig != nil { - kubeConfig, err := imp.WriteKubeConfig(ctx) - if err != nil { - return "", err - } - cmd = fmt.Sprintf("%s --kubeconfig=%s", cmd, kubeConfig) - } else { - // impersonate SA - if kustomization.Spec.ServiceAccountName != "" { - saToken, err := imp.GetServiceAccountToken(ctx) - if err != nil { - return "", fmt.Errorf("service account impersonation failed: %w", err) - } - - cmd = fmt.Sprintf("%s --token %s", cmd, saToken) - } + cmd, err := r.buildKubectlCmdImpersonation(ctx, kustomization, imp, cmd) + if err != nil { + return "", err } command := exec.CommandContext(applyCtx, "/bin/sh", "-c", cmd) @@ -758,7 +736,7 @@ func (r *KustomizationReconciler) reconcileDelete(ctx context.Context, kustomiza log := logr.FromContext(ctx) if kustomization.Spec.Prune && !kustomization.Spec.Suspend { // create any necessary kube-clients - imp := NewKustomizeImpersonation(kustomization, r.Client, r.StatusPoller, "") + imp := NewKustomizeImpersonation(kustomization, r.EnableUserImpersonation, r.Client, r.StatusPoller, "") client, _, err := imp.GetClient(ctx) if err != nil { err = fmt.Errorf("failed to build kube client for Kustomization: %w", err) @@ -865,3 +843,84 @@ func (r *KustomizationReconciler) patchStatus(ctx context.Context, req ctrl.Requ return r.Status().Patch(ctx, &kustomization, patch) } + +func (r *KustomizationReconciler) buildKubectlCmdImpersonation( + ctx context.Context, kustomization kustomizev1.Kustomization, + imp *KustomizeImpersonation, + cmd string, +) (string, error) { + enableUserImpersonation := r.EnableUserImpersonation + if kustomization.Spec.ServiceAccountName != "" && kustomization.Spec.Principal == nil { + enableUserImpersonation = false + kustomization.Spec.Principal = &kustomizev1.Principal{ + Name: kustomization.Spec.ServiceAccountName, + Kind: pkgclient.ServiceAccountType, + } + } + + if kustomization.Spec.KubeConfig != nil { + kubeConfig, err := imp.WriteKubeConfig(ctx) + if err != nil { + return "", err + } + cmd = fmt.Sprintf("%s --kubeconfig=%s", cmd, kubeConfig) + + if !enableUserImpersonation || kustomization.Spec.Principal == nil { + return cmd, nil + } + } + + if enableUserImpersonation { + var user string + if kustomization.Spec.Principal == nil { + user = fmt.Sprintf("flux:user:%s:%s", kustomization.Namespace, pkgclient.DefaultUser) + } else if kustomization.Spec.Principal.Kind == pkgclient.UserType { + user = fmt.Sprintf("flux:user:%s:%s", kustomization.Namespace, kustomization.Spec.Principal.Name) + } else if kustomization.Spec.Principal.Kind == pkgclient.ServiceAccountType { + user = fmt.Sprintf("system:serviceaccount:%s:%s", kustomization.Namespace, kustomization.Spec.Principal.Name) + } else { + return "", errors.New("unknown kind for impersonation, accepted kinds are User and ServiceAccount") + } + + cmd = fmt.Sprintf("%s --as %s", cmd, user) + + if c, err := rest.InClusterConfig(); err == nil && c != nil { + cmd = fmt.Sprintf("%s --certificate-authority=%s --token=%s --server=%s", cmd, c.CAFile, c.BearerToken, c.Host) + } + + if kustomization.Spec.Principal != nil && kustomization.Spec.Principal.Kind == pkgclient.ServiceAccountType { + return cmd, nil + } + + cmd = fmt.Sprintf("%s --as-group flux:users --as-group system:authenticated --as-group flux:users:%s", cmd, kustomization.Namespace) + return cmd, nil + } + + if kustomization.Spec.Principal == nil { + return cmd, nil + } + + var saName string + if kustomization.Spec.ServiceAccountName != "" { + saName = kustomization.Spec.ServiceAccountName + } else if kustomization.Spec.Principal.Kind == pkgclient.ServiceAccountType { + saName = kustomization.Spec.Principal.Name + } else if kustomization.Spec.Principal.Kind == pkgclient.UserType { + return "", errors.New("cannot impersonate user if --user-impersonation is not set") + } else { + return "", errors.New("unknown kind for impersonation, accepted kinds are User and ServiceAccount") + } + + saToken, err := pkgclient.GetServiceAccountToken(ctx, r.Client, pkgclient.ImpersonationConfig{ + Namespace: kustomization.Namespace, + Name: saName, + Kind: pkgclient.ServiceAccountType, + }) + + if err != nil { + return "", fmt.Errorf("service account impersonation failed: %w", err) + } + + cmd = fmt.Sprintf("%s --token %s", cmd, saToken) + return cmd, nil +} diff --git a/controllers/kustomization_impersonation.go b/controllers/kustomization_impersonation.go index c013951b..43da2df6 100644 --- a/controllers/kustomization_impersonation.go +++ b/controllers/kustomization_impersonation.go @@ -20,9 +20,6 @@ import ( "context" "fmt" "io/ioutil" - "strings" - - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/cli-utils/pkg/kstatus/polling" @@ -30,67 +27,32 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/client/config" + pkgclient "github.com/fluxcd/pkg/runtime/client" + kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1" ) type KustomizeImpersonation struct { - workdir string - kustomization kustomizev1.Kustomization - statusPoller *polling.StatusPoller + workdir string + enableUserImpersonation bool + kustomization kustomizev1.Kustomization + statusPoller *polling.StatusPoller client.Client } func NewKustomizeImpersonation( kustomization kustomizev1.Kustomization, + enableUserImpersonation bool, kubeClient client.Client, statusPoller *polling.StatusPoller, workdir string) *KustomizeImpersonation { return &KustomizeImpersonation{ - workdir: workdir, - kustomization: kustomization, - statusPoller: statusPoller, - Client: kubeClient, - } -} - -func (ki *KustomizeImpersonation) GetServiceAccountToken(ctx context.Context) (string, error) { - namespacedName := types.NamespacedName{ - Namespace: ki.kustomization.Namespace, - Name: ki.kustomization.Spec.ServiceAccountName, - } - - var serviceAccount corev1.ServiceAccount - err := ki.Client.Get(ctx, namespacedName, &serviceAccount) - if err != nil { - return "", err - } - - secretName := types.NamespacedName{ - Namespace: ki.kustomization.Namespace, - Name: ki.kustomization.Spec.ServiceAccountName, + workdir: workdir, + kustomization: kustomization, + statusPoller: statusPoller, + Client: kubeClient, + enableUserImpersonation: enableUserImpersonation, } - - for _, secret := range serviceAccount.Secrets { - if strings.HasPrefix(secret.Name, fmt.Sprintf("%s-token", serviceAccount.Name)) { - secretName.Name = secret.Name - break - } - } - - var secret corev1.Secret - err = ki.Client.Get(ctx, secretName, &secret) - if err != nil { - return "", err - } - - var token string - if data, ok := secret.Data["token"]; ok { - token = string(data) - } else { - return "", fmt.Errorf("the service account secret '%s' does not containt a token", secretName.String()) - } - - return token, nil } // GetClient creates a controller-runtime client for talking to a Kubernetes API server. @@ -99,27 +61,42 @@ func (ki *KustomizeImpersonation) GetServiceAccountToken(ctx context.Context) (s // If --kubeconfig is set, will use the kubeconfig file at that location. // Otherwise will assume running in cluster and use the cluster provided kubeconfig. func (ki *KustomizeImpersonation) GetClient(ctx context.Context) (client.Client, *polling.StatusPoller, error) { + clientgenOpts := pkgclient.ImpersonationConfig{ + Namespace: ki.kustomization.Namespace, + Enabled: ki.enableUserImpersonation, + } + + if ki.kustomization.Spec.Principal == nil && ki.kustomization.Spec.ServiceAccountName != "" || + !ki.enableUserImpersonation && ki.kustomization.Spec.ServiceAccountName != "" { + clientgenOpts.Name = ki.kustomization.Spec.ServiceAccountName + clientgenOpts.Kind = pkgclient.ServiceAccountType + clientgenOpts.Enabled = false + } else if ki.kustomization.Spec.Principal != nil { + clientgenOpts.Name = ki.kustomization.Spec.Principal.Name + clientgenOpts.Kind = ki.kustomization.Spec.Principal.Kind + } + if ki.kustomization.Spec.KubeConfig == nil { - if ki.kustomization.Spec.ServiceAccountName != "" { - return ki.clientForServiceAccount(ctx) - } + return ki.clientForImpersonation(ctx, clientgenOpts) + } - return ki.Client, ki.statusPoller, nil + clientgenOpts.KubeConfig = &pkgclient.KubeConfig{ + SecretRef: ki.kustomization.Spec.KubeConfig.SecretRef, } - return ki.clientForKubeConfig(ctx) + return ki.clientForKubeConfig(ctx, clientgenOpts) } -func (ki *KustomizeImpersonation) clientForServiceAccount(ctx context.Context) (client.Client, *polling.StatusPoller, error) { - token, err := ki.GetServiceAccountToken(ctx) +func (ki *KustomizeImpersonation) clientForImpersonation(ctx context.Context, impConfig pkgclient.ImpersonationConfig) (client.Client, *polling.StatusPoller, error) { + restConfig, err := config.GetConfig() if err != nil { return nil, nil, err } - restConfig, err := config.GetConfig() + + restConfig, err = pkgclient.GetConfigForAccount(ctx, ki.Client, restConfig, impConfig) + if err != nil { return nil, nil, err } - restConfig.BearerToken = token - restConfig.BearerTokenFile = "" // Clear, as it overrides BearerToken restMapper, err := apiutil.NewDynamicRESTMapper(restConfig) if err != nil { @@ -133,11 +110,16 @@ func (ki *KustomizeImpersonation) clientForServiceAccount(ctx context.Context) ( statusPoller := polling.NewStatusPoller(client, restMapper) return client, statusPoller, err - } -func (ki *KustomizeImpersonation) clientForKubeConfig(ctx context.Context) (client.Client, *polling.StatusPoller, error) { - kubeConfigBytes, err := ki.getKubeConfig(ctx) +func (ki *KustomizeImpersonation) clientForKubeConfig(ctx context.Context, impConfig pkgclient.ImpersonationConfig) (client.Client, *polling.StatusPoller, error) { + secretName := types.NamespacedName{ + Namespace: ki.kustomization.GetNamespace(), + Name: ki.kustomization.Spec.KubeConfig.SecretRef.Name, + } + + kubeConfigBytes, err := pkgclient.GetKubeConfigFromSecret(ctx, ki.Client, secretName) + if err != nil { return nil, nil, err } @@ -147,6 +129,14 @@ func (ki *KustomizeImpersonation) clientForKubeConfig(ctx context.Context) (clie return nil, nil, err } + // Only impersonate if user impersonation is enabled and the principal is set + if impConfig.Enabled && impConfig.Kind != "" && impConfig.Name != "" { + restConfig, err = pkgclient.GetConfigForAccount(ctx, ki.Client, restConfig, impConfig) + if err != nil { + return nil, nil, err + } + } + restMapper, err := apiutil.NewDynamicRESTMapper(restConfig) if err != nil { return nil, nil, err @@ -168,7 +158,7 @@ func (ki *KustomizeImpersonation) WriteKubeConfig(ctx context.Context) (string, Name: ki.kustomization.Spec.KubeConfig.SecretRef.Name, } - kubeConfig, err := ki.getKubeConfig(ctx) + kubeConfig, err := pkgclient.GetKubeConfigFromSecret(ctx, ki.Client, secretName) if err != nil { return "", err } @@ -183,22 +173,3 @@ func (ki *KustomizeImpersonation) WriteKubeConfig(ctx context.Context) (string, } return f.Name(), nil } - -func (ki *KustomizeImpersonation) getKubeConfig(ctx context.Context) ([]byte, error) { - secretName := types.NamespacedName{ - Namespace: ki.kustomization.GetNamespace(), - Name: ki.kustomization.Spec.KubeConfig.SecretRef.Name, - } - - var secret corev1.Secret - if err := ki.Get(ctx, secretName, &secret); err != nil { - return nil, fmt.Errorf("unable to read KubeConfig secret '%s' error: %w", secretName.String(), err) - } - - kubeConfig, ok := secret.Data["value"] - if !ok { - return nil, fmt.Errorf("KubeConfig secret '%s' doesn't contain a 'value' key ", secretName.String()) - } - - return kubeConfig, nil -} diff --git a/controllers/utils.go b/controllers/utils.go index 809a3037..1e6b8bcd 100644 --- a/controllers/utils.go +++ b/controllers/utils.go @@ -17,9 +17,10 @@ limitations under the License. package controllers import ( + "strings" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" - "strings" ) // parseApplyOutput extracts the objects and the action diff --git a/controllers/utils_test.go b/controllers/utils_test.go index 39b75c77..d96e5063 100644 --- a/controllers/utils_test.go +++ b/controllers/utils_test.go @@ -1,8 +1,17 @@ package controllers import ( + "context" "strings" "testing" + + pkgclient "github.com/fluxcd/pkg/runtime/client" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1" ) func TestParseApplyError(t *testing.T) { @@ -54,3 +63,201 @@ error: error validating data: unknown field "ima ge" in io.k8s.api.core.v1.Cont }) } } + +func TestBuildKubectlCmdImpersonation(t *testing.T) { + tests := []struct { + kustomization kustomizev1.Kustomization + imp *KustomizeImpersonation + contain string + notContain []string + }{ + { + kustomization: kustomizev1.Kustomization{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test", + }, + }, + imp: &KustomizeImpersonation{ + enableUserImpersonation: false, + }, + contain: "", + notContain: []string{"--as", "--token", "--as-group"}, + }, + { + kustomization: kustomizev1.Kustomization{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: kustomizev1.KustomizationSpec{ + ServiceAccountName: "test-sa", + }, + }, + imp: &KustomizeImpersonation{ + enableUserImpersonation: false, + }, + contain: "--token random-token", + notContain: []string{"--as", "--as-group"}, + }, + { + // Use token impersonation once serviceAccountName is set and principal is unset + kustomization: kustomizev1.Kustomization{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: kustomizev1.KustomizationSpec{ + ServiceAccountName: "test-sa", + }, + }, + imp: &KustomizeImpersonation{ + enableUserImpersonation: true, + }, + contain: "--token random-token", + notContain: []string{"--as", "--as-group"}, + }, + { + // Should use token impersonation if --user-impersonation is disabled + // and both serviceAccountName and principal + kustomization: kustomizev1.Kustomization{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: kustomizev1.KustomizationSpec{ + ServiceAccountName: "test-sa", + Principal: &kustomizev1.Principal{ + Kind: pkgclient.UserType, + Name: "dev", + }, + }, + }, + imp: &KustomizeImpersonation{ + enableUserImpersonation: false, + }, + contain: "--token random-token", + notContain: []string{"--as", "--as-group"}, + }, + { + // Should use user impersonation if --user-impersonation is enabled + // and both serviceAccountName and principal are set + kustomization: kustomizev1.Kustomization{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test", + }, + Spec: kustomizev1.KustomizationSpec{ + ServiceAccountName: "test-sa", + Principal: &kustomizev1.Principal{ + Kind: pkgclient.UserType, + Name: "dev", + }, + }, + }, + imp: &KustomizeImpersonation{ + enableUserImpersonation: true, + }, + contain: "--as flux:user:test:dev", + notContain: []string{"--token"}, + }, + { + kustomization: kustomizev1.Kustomization{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test", + }, + Spec: kustomizev1.KustomizationSpec{ + Principal: &kustomizev1.Principal{ + Kind: pkgclient.UserType, + Name: "dev", + }, + }, + }, + imp: &KustomizeImpersonation{ + enableUserImpersonation: true, + }, + contain: "--as flux:user:test:dev", + notContain: []string{"--token"}, + }, + { + kustomization: kustomizev1.Kustomization{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test", + }, + }, + imp: &KustomizeImpersonation{ + enableUserImpersonation: true, + }, + contain: "--as flux:user:test:reconciler", + notContain: []string{"--token"}, + }, + { + kustomization: kustomizev1.Kustomization{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test", + }, + Spec: kustomizev1.KustomizationSpec{ + Principal: &kustomizev1.Principal{ + Kind: pkgclient.ServiceAccountType, + Name: "dev", + }, + }, + }, + imp: &KustomizeImpersonation{ + enableUserImpersonation: true, + }, + contain: "--as system:serviceaccount:test:dev", + notContain: []string{"--token"}, + }, + } + + for _, tt := range tests { + ctx := context.Background() + fakeClient := setupFakeClient() + r := KustomizationReconciler{ + Client: fakeClient, + } + r.EnableUserImpersonation = tt.imp.enableUserImpersonation + cmd, err := r.buildKubectlCmdImpersonation(ctx, tt.kustomization, tt.imp, "") + if err != nil { + t.Fatalf("error while building cmd for impersonation: %s", err) + } + + if !strings.Contains(cmd, tt.contain) { + t.Errorf("impersonation cmd %q should contain substring %q", + cmd, tt.contain) + } + + for _, str := range tt.notContain { + if strings.Contains(cmd, str) { + t.Errorf("impersonation cmd %q shouldn't contain %q", + cmd, str) + } + } + } +} + +func setupFakeClient() client.Client { + clientBuilder := fake.NewClientBuilder() + serviceaccount := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sa", + }, + Secrets: []corev1.ObjectReference{ + { + Name: "test-sa-token", + }, + }, + } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sa-token", + }, + Data: map[string][]byte{ + "token": []byte("random-token"), + }, + } + clientBuilder.WithRuntimeObjects(serviceaccount, secret) + + return clientBuilder.Build() +} diff --git a/docs/api/kustomize.md b/docs/api/kustomize.md index 84f1cb06..aefae22c 100644 --- a/docs/api/kustomize.md +++ b/docs/api/kustomize.md @@ -336,6 +336,21 @@ bool when patching fails due to an immutable field change.

+ + +principal
+ + +Principal + + + + +

Principal provides details on how the controller should +carry out impersonation during garbage collections, health-check +and applies

+ + @@ -795,6 +810,21 @@ bool when patching fails due to an immutable field change.

+ + +principal
+ + +Principal + + + + +

Principal provides details on how the controller should +carry out impersonation during garbage collections, health-check +and applies

+ + @@ -955,6 +985,49 @@ must match the vars declared in the manifests for the substitution to happen.

+

Principal +

+

+(Appears on: +KustomizationSpec) +

+
+
+ + + + + + + + + + + + + + + + + +
FieldDescription
+kind
+ +string + +		 +

Kind specifies the kind of object to be impersonated +The kind could be ‘User’ or ‘ServiceAccount’

+
+name
+ +string + +		 +

The name of the object to be impersonated

+
+
+

Snapshot

diff --git a/go.mod b/go.mod index 79ba37dd..af5e9a26 100644 --- a/go.mod +++ b/go.mod @@ -39,3 +39,5 @@ require ( sigs.k8s.io/kustomize/api v0.7.4 sigs.k8s.io/yaml v1.2.0 ) + +replace github.com/fluxcd/pkg/runtime v0.11.0 => github.com/SomtochiAma/pkg/runtime v0.6.1-0.20210604132416-b8a19e82b555 diff --git a/go.sum b/go.sum index c2a8ec82..240f280f 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/SomtochiAma/pkg/runtime v0.6.1-0.20210604132416-b8a19e82b555 h1:Yof/u5vOFKB3BzXGSvD6jkmYNUmrNSTNbeeJByV5JQQ= +github.com/SomtochiAma/pkg/runtime v0.6.1-0.20210604132416-b8a19e82b555/go.mod h1:ZjAwug6DBLXwo9UdP1/tTPyuWpK9kZ0BEJbctbuEB1o= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -198,8 +200,6 @@ github.com/fluxcd/pkg/apis/kustomize v0.0.1 h1:TkA80R0GopRY27VJqzKyS6ifiKIAfwBd7 github.com/fluxcd/pkg/apis/kustomize v0.0.1/go.mod h1:JAFPfnRmcrAoG1gNiA8kmEXsnOBuDyZ/F5X4DAQcVV0= github.com/fluxcd/pkg/apis/meta v0.9.0 h1:rxW69p+VmJCKXXkaRYnovRBFlKjd+MJQfm2RrB0B4j8= github.com/fluxcd/pkg/apis/meta v0.9.0/go.mod h1:yHuY8kyGHYz22I0jQzqMMGCcHViuzC/WPdo9Gisk8Po= -github.com/fluxcd/pkg/runtime v0.11.0 h1:FPsiu1k5NQGl2tsaXH5WgSmrOMg7o44jdOP0rW/TI1Y= -github.com/fluxcd/pkg/runtime v0.11.0/go.mod h1:ZjAwug6DBLXwo9UdP1/tTPyuWpK9kZ0BEJbctbuEB1o= github.com/fluxcd/pkg/testserver v0.0.2 h1:SoaMtO9cE5p/wl2zkGudzflnEHd9mk68CGjZOo7w0Uk= github.com/fluxcd/pkg/testserver v0.0.2/go.mod h1:pgUZTh9aQ44FSTQo+5NFlh7YMbUfdz1B80DalW7k96Y= github.com/fluxcd/pkg/untar v0.0.5 h1:UGI3Ch1UIEIaqQvMicmImL1s9npQa64DJ/ozqHKB7gk= diff --git a/main.go b/main.go index f89c43c1..ac373be5 100644 --- a/main.go +++ b/main.go @@ -60,16 +60,17 @@ func init() { func main() { var ( - metricsAddr string - eventsAddr string - healthAddr string - concurrent int - requeueDependency time.Duration - clientOptions client.Options - logOptions logger.Options - leaderElectionOptions leaderelection.Options - watchAllNamespaces bool - httpRetry int + metricsAddr string + eventsAddr string + healthAddr string + concurrent int + requeueDependency time.Duration + clientOptions client.Options + logOptions logger.Options + leaderElectionOptions leaderelection.Options + watchAllNamespaces bool + httpRetry int + enableUserImpersonation bool ) flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.") @@ -79,6 +80,8 @@ func main() { flag.DurationVar(&requeueDependency, "requeue-dependency", 30*time.Second, "The interval at which failing dependencies are reevaluated.") flag.BoolVar(&watchAllNamespaces, "watch-all-namespaces", true, "Watch for custom resources in all namespaces, if set to false it will only watch the runtime namespace.") + flag.BoolVar(&enableUserImpersonation, "user-impersonation", false, + "Use user impersonation instead of service account token impersonation") flag.IntVar(&httpRetry, "http-retry", 9, "The maximum number of retries when failing to fetch artifacts over HTTP.") clientOptions.BindFlags(flag.CommandLine) logOptions.BindFlags(flag.CommandLine) @@ -129,12 +132,13 @@ func main() { pprof.SetupHandlers(mgr, setupLog) if err = (&controllers.KustomizationReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - EventRecorder: mgr.GetEventRecorderFor(controllerName), - ExternalEventRecorder: eventRecorder, - MetricsRecorder: metricsRecorder, - StatusPoller: polling.NewStatusPoller(mgr.GetClient(), mgr.GetRESTMapper()), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + EventRecorder: mgr.GetEventRecorderFor(controllerName), + ExternalEventRecorder: eventRecorder, + MetricsRecorder: metricsRecorder, + StatusPoller: polling.NewStatusPoller(mgr.GetClient(), mgr.GetRESTMapper()), + EnableUserImpersonation: enableUserImpersonation, }).SetupWithManager(mgr, controllers.KustomizationReconcilerOptions{ MaxConcurrentReconciles: concurrent, DependencyRequeueInterval: requeueDependency,