Merge pull request #343 from fluxcd/kustomize-v4

Upgrade controller to Kustomize v4

.github/workflows/nightly.yml

@@ -5,7 +5,7 @@ on:
  workflow_dispatch:
 
 env:
- CONTROLLER: ${{ github.event.repository.name }}
+ REPOSITORY: ${{ github.repository }}
 
 jobs:
  build:
@@ -30,4 +30,4 @@ jobs:
  file: ./Dockerfile
  platforms: linux/amd64,linux/arm/v7,linux/arm64
  tags: |
- fluxcd/${{ env.CONTROLLER }}:nightly
+ ${{ env.REPOSITORY }}:nightly

Makefile

@@ -2,7 +2,7 @@
 IMG ?= fluxcd/kustomize-controller:latest
 # Produce CRDs that work back to Kubernetes 1.16
 CRD_OPTIONS ?= crd:crdVersions=v1
-SOURCE_VER ?= v0.13.2
+SOURCE_VER ?= v0.14.0
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -107,7 +107,7 @@ ifeq (, $(shell which controller-gen))
  CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
  cd $$CONTROLLER_GEN_TMP_DIR ;\
  go mod init tmp ;\
- go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1 ;\
+ go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.5.0 ;\
  rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
  }
 CONTROLLER_GEN=$(GOBIN)/controller-gen

api/go.mod

@@ -3,10 +3,10 @@ module github.com/fluxcd/kustomize-controller/api
 go 1.16
 
 require (
- github.com/fluxcd/pkg/apis/kustomize v0.0.1
- github.com/fluxcd/pkg/apis/meta v0.9.0
- github.com/fluxcd/pkg/runtime v0.11.0
- k8s.io/apiextensions-apiserver v0.20.4
- k8s.io/apimachinery v0.20.4
- sigs.k8s.io/controller-runtime v0.8.3
+ github.com/fluxcd/pkg/apis/kustomize v0.1.0
+ github.com/fluxcd/pkg/apis/meta v0.10.0
+ github.com/fluxcd/pkg/runtime v0.12.0
+ k8s.io/apiextensions-apiserver v0.21.1
+ k8s.io/apimachinery v0.21.1
+ sigs.k8s.io/controller-runtime v0.9.0
 )

config/default/kustomization.yaml

@@ -2,8 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: kustomize-system
 resources:
-- https://github.com/fluxcd/source-controller/releases/download/v0.13.2/source-controller.crds.yaml
-- https://github.com/fluxcd/source-controller/releases/download/v0.13.2/source-controller.deployment.yaml
+- https://github.com/fluxcd/source-controller/releases/download/v0.14.0/source-controller.crds.yaml
+- https://github.com/fluxcd/source-controller/releases/download/v0.14.0/source-controller.deployment.yaml
 - ../crd
 - ../rbac
 - ../manager

controllers/kustomization_controller_test.go

@@ -36,8 +36,6 @@ import (
  apimeta "k8s.io/apimachinery/pkg/api/meta"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  "k8s.io/apimachinery/pkg/types"
- "k8s.io/client-go/tools/clientcmd"
- clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
  "sigs.k8s.io/controller-runtime/pkg/client"
 
  "github.com/fluxcd/pkg/apis/meta"
@@ -366,29 +364,12 @@ spec:
 })
 
 func kubeConfigSecret() (*corev1.Secret, error) {
- c := clientcmdapi.NewConfig()
- c.CurrentContext = "default"
- c.Clusters["default"] = &clientcmdapi.Cluster{
- Server: cfg.Host,
- }
- c.Contexts["default"] = &clientcmdapi.Context{
- Cluster: "default",
- Namespace: "default",
- AuthInfo: "default",
- }
- c.AuthInfos["default"] = &clientcmdapi.AuthInfo{
- Token: cfg.BearerToken,
- }
- cb, err := clientcmd.Write(*c)
- if err != nil {
- return nil, err
- }
  return &corev1.Secret{
  ObjectMeta: metav1.ObjectMeta{
  Name: "kubeconfig",
  },
  Data: map[string][]byte{
- "value": cb,
+ "value": kubeConfig,
  },
  }, nil
 }

controllers/kustomization_generator.go

@@ -25,12 +25,13 @@ import (
  "os"
  "path/filepath"
  "strings"
+ "sync"
 
  "sigs.k8s.io/controller-runtime/pkg/client"
  "sigs.k8s.io/kustomize/api/filesys"
- "sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
  "sigs.k8s.io/kustomize/api/konfig"
  "sigs.k8s.io/kustomize/api/krusty"
+ "sigs.k8s.io/kustomize/api/provider"
  "sigs.k8s.io/kustomize/api/resmap"
  kustypes "sigs.k8s.io/kustomize/api/types"
  "sigs.k8s.io/yaml"
@@ -162,7 +163,8 @@ func (kg *KustomizeGenerator) generateKustomization(dirPath string) error {
 
  scan := func(base string) ([]string, error) {
  var paths []string
- uf := kunstruct.NewKunstructuredFactoryImpl()
+ pvd := provider.NewDefaultDepProvider()
+ rf := pvd.GetResourceFactory()
  err := fs.Walk(base, func(path string, info os.FileInfo, err error) error {
  if err != nil {
  return err
@@ -192,7 +194,7 @@ func (kg *KustomizeGenerator) generateKustomization(dirPath string) error {
  return err
  }
 
- if _, err := uf.SliceFromBytes(fContents); err != nil {
+ if _, err := rf.SliceFromBytes(fContents); err != nil {
  return fmt.Errorf("failed to decode Kubernetes YAML from %s: %w", path, err)
  }
  paths = append(paths, path)
@@ -332,25 +334,27 @@ func adaptSelector(selector *kustomize.Selector) (output *kustypes.Selector) {
  return
 }
 
+// TODO: remove mutex when kustomize fixes the concurrent map read/write panic
+var kustomizeBuildMutex sync.Mutex
+
 // buildKustomization wraps krusty.MakeKustomizer with the following settings:
-// - disable kyaml due to critical bugs like:
-// - https://github.com/kubernetes-sigs/kustomize/issues/3446
-// - https://github.com/kubernetes-sigs/kustomize/issues/3480
 // - reorder the resources just before output (Namespaces and Cluster roles/role bindings first, CRDs before CRs, Webhooks last)
 // - load files from outside the kustomization.yaml root
 // - disable plugins except for the builtin ones
-// - prohibit changes to resourceIds, patch name/kind don't overwrite target name/kind
 func buildKustomization(fs filesys.FileSystem, dirPath string) (resmap.ResMap, error) {
+ // temporary workaround for concurrent map read and map write bug
+ // https://github.com/kubernetes-sigs/kustomize/issues/3659
+ kustomizeBuildMutex.Lock()
+ defer kustomizeBuildMutex.Unlock()
+
  buildOptions := &krusty.Options{
- UseKyaml: false,
- DoLegacyResourceSort: true,
- LoadRestrictions: kustypes.LoadRestrictionsNone,
- AddManagedbyLabel: false,
- DoPrune: false,
- PluginConfig: konfig.DisabledPluginConfig(),
- AllowResourceIdChanges: false,
+ DoLegacyResourceSort: true,
+ LoadRestrictions: kustypes.LoadRestrictionsNone,
+ AddManagedbyLabel: false,
+ DoPrune: false,
+ PluginConfig: kustypes.DisabledPluginConfig(),
  }
 
- k := krusty.MakeKustomizer(fs, buildOptions)
- return k.Run(dirPath)
+ k := krusty.MakeKustomizer(buildOptions)
+ return k.Run(fs, dirPath)
 }

controllers/suite_test.go

@@ -46,6 +46,7 @@ var cfg *rest.Config
 var k8sClient client.Client
 var k8sManager ctrl.Manager
 var testEnv *envtest.Environment
+var kubeConfig []byte
 
 func TestAPIs(t *testing.T) {
  RegisterFailHandler(Fail)
@@ -77,6 +78,15 @@ var _ = BeforeSuite(func(done Done) {
  Expect(err).ToNot(HaveOccurred())
  Expect(cfg).ToNot(BeNil())
 
+ user, err := testEnv.ControlPlane.AddUser(envtest.User{
+ Name: "envtest-admin",
+ Groups: []string{"system:masters"},
+ }, nil)
+ Expect(err).ToNot(HaveOccurred())
+
+ kubeConfig, err = user.KubeConfig()
+ Expect(err).ToNot(HaveOccurred())
+
  err = kustomizev1.AddToScheme(scheme.Scheme)
  Expect(err).NotTo(HaveOccurred())
 

go.mod

@@ -6,36 +6,38 @@ replace github.com/fluxcd/kustomize-controller/api => ./api
 
 require (
  filippo.io/age v1.0.0-beta7
- github.com/Microsoft/go-winio v0.4.16 // indirect
  github.com/cyphar/filepath-securejoin v0.2.2
  github.com/drone/envsubst v1.0.3-0.20200804185402-58bc65f69603
  github.com/fluxcd/kustomize-controller/api v0.12.2
- github.com/fluxcd/pkg/apis/kustomize v0.0.1
- github.com/fluxcd/pkg/apis/meta v0.9.0
- github.com/fluxcd/pkg/runtime v0.11.0
- github.com/fluxcd/pkg/testserver v0.0.2
- github.com/fluxcd/pkg/untar v0.0.5
- github.com/fluxcd/source-controller/api v0.13.2
- github.com/go-logr/logr v0.3.0
+ github.com/fluxcd/pkg/apis/kustomize v0.1.0
+ github.com/fluxcd/pkg/apis/meta v0.10.0
+ github.com/fluxcd/pkg/runtime v0.12.0
+ github.com/fluxcd/pkg/testserver v0.1.0
+ github.com/fluxcd/pkg/untar v0.1.0
+ github.com/fluxcd/source-controller/api v0.14.0
+ github.com/go-logr/logr v0.4.0
  github.com/hashicorp/go-retryablehttp v0.6.8
  github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c
- github.com/imdario/mergo v0.3.12 // indirect
- github.com/onsi/ginkgo v1.14.2
- github.com/onsi/gomega v1.10.2
+ github.com/onsi/ginkgo v1.16.4
+ github.com/onsi/gomega v1.13.0
  github.com/spf13/pflag v1.0.5
  go.mozilla.org/gopgagent v0.0.0-20170926210634-4d7ea76ff71a
  go.mozilla.org/sops/v3 v3.7.1
  golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
- golang.org/x/net v0.0.0-20210326060303-6b1517762897
+ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781
  google.golang.org/grpc v1.27.1
- gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
- k8s.io/api v0.20.4
- k8s.io/apiextensions-apiserver v0.20.4
- k8s.io/apimachinery v0.20.4
- k8s.io/cli-runtime v0.20.4 // indirect
- k8s.io/client-go v0.20.4
- sigs.k8s.io/cli-utils v0.22.4
- sigs.k8s.io/controller-runtime v0.8.3
- sigs.k8s.io/kustomize/api v0.7.4
+ k8s.io/api v0.21.1
+ k8s.io/apiextensions-apiserver v0.21.1
+ k8s.io/apimachinery v0.21.1
+ k8s.io/client-go v0.21.1
+ sigs.k8s.io/cli-utils v0.25.1-0.20210608181808-f3974341173a
+ sigs.k8s.io/controller-runtime v0.9.0
+ sigs.k8s.io/kustomize/api v0.8.10
  sigs.k8s.io/yaml v1.2.0
 )
+
+// pin kustomize to v4.1.3
+replace (
+ sigs.k8s.io/kustomize/api => sigs.k8s.io/kustomize/api v0.8.10
+ sigs.k8s.io/kustomize/kyaml => sigs.k8s.io/kustomize/kyaml v0.10.21
+)