Install ACM on the Kubeflow CI management cluster.

* We want to install ACM on the kf-ci-management cluster in project
  kubeflow-ci so that we can start using GitOps to manage CI infrastructure.

* Related to kubeflow#644

* Remove status from the cleanup ci job. This breaks ACM sync.

* Add a cluster selector to ACM so that we only install the auto-deploy
  namespace on the appropriate cluster.

* Add an annotation to all auto-deploy tasks so we only synchronize them to the appropriate cluster.

acm-repo/clusterregistry/kf-ci-selector.yaml

@@ -0,0 +1,10 @@
+# A cluster selector for our v1 ci clusters.
+# These are clusters running CI infrasture with tekton installed.
+kind: ClusterSelector
+apiVersion: configmanagement.gke.io/v1
+metadata:
+  name: kf-ci-tekton
+spec:
+  selector:
+    matchLabels:
+      environment: kf-ci-tekton

acm-repo/namespaces/auto-deploy/batch_v1beta1_cronjob_cleanup-ci-kubeflow-ci-deployment.yaml

@@ -45,6 +45,4 @@ spec:
             name: cleanup-config
   schedule: 0 */2 * * *
   successfulJobsHistoryLimit: 3
-  suspend: false
-status:
-  lastScheduleTime: "2020-05-07T14:00:00Z"
+  suspend: false

acm-repo/namespaces/auto-deploy/namespace.yaml

@@ -1,4 +1,8 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: auto-deploy  
+  name: auto-deploy
+  annotations:
+    # Auto-deploy namespace should only be created in CI clusters
+    # running tekton.
+    configmanagement.gke.io/cluster-selector: kf-ci-tekton

acm-repo/namespaces/auto-deploy/tekton.dev_v1alpha1_pipeline_deploy-gcp-blueprint.yaml

@@ -1,6 +1,8 @@
 apiVersion: tekton.dev/v1alpha1
 kind: Pipeline
 metadata:
+  annotations:
+    configmanagement.gke.io/cluster-selector: kf-ci-tekton
   name: deploy-gcp-blueprint
   namespace: auto-deploy
 spec:

acm-repo/namespaces/auto-deploy/tekton.dev_v1alpha1_task_cleanup-kubeflow-ci.yaml

@@ -2,6 +2,7 @@ apiVersion: tekton.dev/v1alpha1
 kind: Task
 metadata:
   annotations:
+    configmanagement.gke.io/cluster-selector: kf-ci-tekton
     sidecar.istio.io/inject: "false"
   name: cleanup-kubeflow-ci
   namespace: auto-deploy

acm-repo/namespaces/auto-deploy/tekton.dev_v1alpha1_task_deploy-gcp-blueprint.yaml

@@ -2,6 +2,7 @@ apiVersion: tekton.dev/v1alpha1
 kind: Task
 metadata:
   annotations:
+    configmanagement.gke.io/cluster-selector: kf-ci-tekton
     sidecar.istio.io/inject: "false"
   name: deploy-gcp-blueprint
   namespace: auto-deploy

acm-repo/namespaces/auto-deploy/tekton.dev_v1alpha1_task_notebook-test.yaml

@@ -2,6 +2,7 @@ apiVersion: tekton.dev/v1alpha1
 kind: Task
 metadata:
   annotations:
+    configmanagement.gke.io/cluster-selector: kf-ci-tekton
     sidecar.istio.io/inject: "false"
   name: notebook-test
   namespace: auto-deploy

tekton/templates/installs/auto-deploy/kustomization.yaml

@@ -1,6 +1,10 @@
 # Configure the tekton tasks for the auto-deploy namespace
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
+commonAnnotations:
+  # TODO(jlewi): Do we really need to add this annotation to every resource?
+  # Or is it sufficient to add it to the namespace?
+  configmanagement.gke.io/cluster-selector: kf-ci-tekton
 namespace: auto-deploy
 resources:
 - ../../tasks

test-infra/cleanup/cleanup-ci-cronjob.yaml

@@ -52,5 +52,3 @@ spec:
   schedule: 0 */2 * * *  
   successfulJobsHistoryLimit: 3
   suspend: false
-status:
-  lastScheduleTime: "2020-05-07T14:00:00Z"

test-infra/configsync/config-management-operator.yaml

@@ -0,0 +1,258 @@
+# ----- configmanagement_v1_configmanagement.yaml -----
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  creationTimestamp: null
+  labels:
+    controller-tools.k8s.io: "1.0"
+  name: configmanagements.configmanagement.gke.io
+spec:
+  group: configmanagement.gke.io
+  names:
+    kind: ConfigManagement
+    plural: configmanagements
+  scope: Cluster
+  # NOTE TO MAINTAINERS: controller-gen will try to remove these and
+  # replace it with `version: v1`.  Don't let that happen, see
+  # https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definition-versioning
+  versions:
+  - name: v1
+    served: true
+    storage: true
+  validation:
+    openAPIV3Schema:
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          type: object
+        spec:
+          properties:
+            clusterName:
+              description: ClusterName, if defined, sets the name for this cluster.  If
+                unset, the cluster is considered to be unnamed, and cannot use ClusterSelectors.
+              type: string
+            configConnector:
+              description: ConfigConnector deploys the GCP Config Connector components
+                as recognized by the "cnrm.cloud.google.com/system" label set to "true".
+              properties:
+                enabled:
+                  description: 'Enable or disable the Config Connector.  Default:
+                    false.'
+                  type: boolean
+              type: object
+            git:
+              description: Git contains configuration specific to importing policies
+                from a Git repo.
+              properties:
+                policyDir:
+                  description: 'PolicyDir is the absolute path of the directory that
+                    contains the local policy.  Default: the root directory of the
+                    repo.'
+                  type: string
+                proxy:
+                  description: Proxy is a struct that contains options for configuring
+                    access to the Git repo via a proxy. Only has an effect when secretType
+                    is one of ("cookiefile", "none").  Optional.
+                  properties:
+                    httpProxy:
+                      description: HTTPProxy defines a HTTP_PROXY env variable used
+                        to access the Git repo.  If both HTTPProxy and HTTPSProxy
+                        are specified, HTTPProxy will be ignored. Optional.
+                      type: string
+                    httpsProxy:
+                      description: HTTPSProxy defines a HTTPS_PROXY env variable used
+                        to access the Git repo.  If both HTTPProxy and HTTPSProxy
+                        are specified, HTTPProxy will be ignored. Optional.
+                      type: string
+                  type: object
+                secretType:
+                  description: SecretType is the type of secret configured for access
+                    to the Git repo. Must be one of ssh, cookiefile, gcenode, token,
+                    or none. Required. The validation of this is case-sensitive.
+                  pattern: ^(ssh|cookiefile|gcenode|token|none)$
+                  type: string
+                syncBranch:
+                  description: 'SyncBranch is the branch to sync from.  Default: "master".'
+                  type: string
+                syncRepo:
+                  pattern: ^(((https?|git|ssh):\/\/)|git@)
+                  type: string
+                syncRev:
+                  description: 'SyncRev is the git revision (tag or hash) to check
+                    out. Default: HEAD.'
+                  type: string
+                syncWait:
+                  description: 'SyncWaitSeconds is the time duration in seconds between
+                    consecutive syncs.  Default: 15 seconds. Note that SyncWaitSecs
+                    is not a time.Duration on purpose. This provides a reminder to
+                    developers that customers specify this value using using integers
+                    like "3" in their ConfigManagement YAML. However, time.Duration
+                    is at a nanosecond granularity, and it''s easy to introduce a
+                    bug where it looks like the code is dealing with seconds but its
+                    actually nanoseconds (or vice versa).'
+                  format: int64
+                  type: integer
+              type: object
+            policyController:
+              description: Policy Controller enables PolicyController components as
+                recognized by the "gatekeeper.sh/manifest" label set to "true".
+              properties:
+                auditIntervalSeconds:
+                  description: AuditIntervalSeconds. The number of seconds between
+                    audit runs. Defaults to 60 seconds. To disable audit, set this
+                    to 0.
+                  format: int64
+                  type: integer
+                enabled:
+                  description: 'Enable or disable the Policy Controller.  Default:
+                    false.'
+                  type: boolean
+                exemptableNamespaces:
+                  description: ExemptableNamespaces. The namespaces in this list are
+                    able to have the admission.gatekeeper.sh/ignore label set. When
+                    the label is set, Policy Controller will not be called for that
+                    namespace or any resources contained in it. `gatekeeper-system`
+                    is always exempted.
+                  items:
+                    type: string
+                  type: array
+                referentialRulesEnabled:
+                  description: 'ReferentialRulesEnabled.  If true, Policy Controller
+                    will allow `data.inventory` references in the contents of ConstraintTemplate
+                    Rego.  No effect unless policyController is enabled.  Default:
+                    false.'
+                  type: boolean
+                templateLibraryInstalled:
+                  description: 'TemplateLibraryInstalled.  If true, a set of default
+                    ConstraintTemplates will be deployed to the cluster. ConstraintTemplates
+                    will not be deployed if this is explicitly set to false or if
+                    policyController is not enabled. Default: true.'
+                  type: boolean
+              type: object
+            sourceFormat:
+              description: SourceFormat specifies how the repository is formatted.
+                See documentation for specifics of what these options do.  Must be
+                one of hierarchy, unstructured. Optional. Set to hierarchy if not
+                specified.  The validation of this is case-sensitive.
+              pattern: ^(hierarchy|unstructured|)$
+              type: string
+          type: object
+        status:
+          properties:
+            configManagementVersion:
+              description: ConfigManagementVersion is the semantic version number
+                of the config management system enforced by the currently running
+                config management operator.
+              type: string
+            errors:
+              items:
+                type: string
+              type: array
+            healthy:
+              type: boolean
+          required:
+          - healthy
+          type: object
+      required:
+      - metadata
+      - spec
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+---
+# ----- addons_rolebinding_rbac.yaml -----
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    k8s-app: config-management-operator
+  name: config-management-operator
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: config-management-operator
+subjects:
+- kind: ServiceAccount
+  name: config-management-operator
+  namespace: kube-system
+---
+# ----- addons_role_rbac.yaml -----
+# The Nomos system creates RBAC rules, so it requires
+# full cluster-admin access. Thus, the operator needs
+# to be able to grant tha permission to the installed
+# Nomos components.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    k8s-app: config-management-operator
+  name: config-management-operator
+rules:
+- apiGroups: ["*"]
+  resources: ["*"]
+  verbs: ["*"]
+---
+# ----- manager.yaml -----
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-app: config-management-operator
+  name: config-management-operator
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: config-management-operator
+  namespace: kube-system
+  labels:
+    k8s-app: config-management-operator
+spec:
+  strategy:
+    type: Recreate
+    # must be null due to 3-way merge, as
+    # rollingUpdate added to the resource by default by the APIServer
+    rollingUpdate: null
+  selector:
+    matchLabels:
+      k8s-app: config-management-operator
+      component: config-management-operator
+  template:
+    metadata:
+      labels:
+        k8s-app: config-management-operator
+        component: config-management-operator
+    spec:
+      containers:
+      - command:
+        - /manager
+        - --private-registry=
+        name: manager
+        image: gcr.io/config-management-release/config-management-operator:20200409021017-op
+        resources:
+          requests:
+            cpu: 100m
+            memory: 20Mi
+      serviceAccount: config-management-operator
+---
+# ----- namespace.yaml -----
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: config-management-system
+  labels:
+    configmanagement.gke.io/system: "true"

test-infra/configsync/config-management.yaml

@@ -0,0 +1,22 @@
+apiVersion: configmanagement.gke.io/v1
+kind: ConfigManagement
+metadata:
+  name: config-management
+  annotations:
+    gke.io/cluster: "gke://kubeflow-ci/us-central1/kf-ci-management" # {"type":"string","x-kustomize":{"setBy":"kpt","partialSetters":[{"name":"gcloud.core.project","value":"kubeflow-ci"},{"name":"name","value":"kf-ci-management"},{"name":"location","value":"us-central1"}]}}
+spec:
+  clusterName: "gke://kubeflow-ci/us-central1/kf-ci-management" # {"type":"string","x-kustomize":{"setBy":"kpt","partialSetters":[{"name":"gcloud.core.project","value":"kubeflow-ci"},{"name":"name","value":"kf-ci-management"},{"name":"location","value":"us-central1"}]}}
+  git:
+    # TODO(jlewi): We should change this to branch master on kubeflow/testing
+    # syncRepo: "https://github.com/kubeflow/testing.git" # {"type":"string","x-kustomize":{"setBy":"kpt","partialSetters":[{"name":"source_repo","value":"https://github.com/jlewi/community-infra.git"}]}}
+    syncRepo: "https://github.com/jlewi/testing.git" # {"type":"string","x-kustomize":{"setBy":"kpt","partialSetters":[{"name":"source_repo","value":"https://github.com/jlewi/community-infra.git"}]}}    
+    syncBranch: acm
+    secretType: none
+    policyDir: "/acm-repo" # {"$ref":"#/definitions/io.k8s.cli.setters.sync-repo-dir"}  
+  # Set to true to install and enable Config Connector
+  # We currently don't use ACM to install config connector because
+  # 1. ACM 1.3 was installing a version of KCC which is too old
+  # 2. We already have KCC installed on the management cluster.
+  configConnector:
+    enabled: false
+  sourceFormat: hierarchy