Generate a schema for the ProwJob crd

This PR adds codegeneration that uses controller-gen to generate a CRD schema for ProwJobs based on the go types. This provides us with two advantages: * It is not possible anymore to submit jobs with an invalid schema. Doing this could result in Prow going down completely. * `kubectl explain` can be used to get info about the fields in a prowjob
kubernetes · Feb 4, 2021 · ed86605 · ed86605
1 parent 3c08a01
commit ed86605
Show file tree

Hide file tree

Showing 11 changed files with 10,525 additions and 474 deletions.
diff --git a/config/prow/cluster/prowjob_customresourcedefinition.yaml b/config/prow/cluster/prowjob_customresourcedefinition.yaml
diff --git a/config/prow/cluster/starter-gcs.yaml b/config/prow/cluster/starter-gcs.yaml
@@ -127,93 +127,6 @@ data:
         - image: alpine
           command: ["/bin/date"]
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: prowjobs.prow.k8s.io
-spec:
-  group: prow.k8s.io
-  version: v1
-  names:
-    kind: ProwJob
-    singular: prowjob
-    plural: prowjobs
-  scope: Namespaced
-  validation:
-    openAPIV3Schema:
-      properties:
-        spec:
-          properties:
-            max_concurrency:
-              type: integer
-              minimum: 0
-            type:
-              type: string
-              enum:
-              - "presubmit"
-              - "postsubmit"
-              - "periodic"
-              - "batch"
-        status:
-          properties:
-            state:
-              type: string
-              enum:
-              - "triggered"
-              - "pending"
-              - "success"
-              - "failure"
-              - "aborted"
-              - "error"
-          anyOf:
-          - not:
-              properties:
-                state:
-                  type: string
-                  enum:
-                  - "success"
-                  - "failure"
-                  - "error"
-          - required:
-            - completionTime
-  additionalPrinterColumns:
-  - name: Job
-    type: string
-    description: The name of the job being run.
-    JSONPath: .spec.job
-  - name: BuildId
-    type: string
-    description: The ID of the job being run.
-    JSONPath: .status.build_id
-  - name: Type
-    type: string
-    description: The type of job being run.
-    JSONPath: .spec.type
-  - name: Org
-    type: string
-    description: The org for which the job is running.
-    JSONPath: .spec.refs.org
-  - name: Repo
-    type: string
-    description: The repo for which the job is running.
-    JSONPath: .spec.refs.repo
-  - name: Pulls
-    type: string
-    description: The pulls for which the job is running.
-    JSONPath: ".spec.refs.pulls[*].number"
-  - name: StartTime
-    type: date
-    description: When the job started running.
-    JSONPath: .status.startTime
-  - name: CompletionTime
-    type: date
-    description: When the job finished running.
-    JSONPath: .status.completionTime
-  - name: State
-    description: The state of the job.
-    type: string
-    JSONPath: .status.state
----
 apiVersion: apps/v1
 kind: Deployment
 metadata:

diff --git a/config/prow/cluster/starter-s3.yaml b/config/prow/cluster/starter-s3.yaml
@@ -126,93 +126,6 @@ data:
         - image: alpine
           command: ["/bin/date"]
 ---
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: prowjobs.prow.k8s.io
-spec:
-  group: prow.k8s.io
-  version: v1
-  names:
-    kind: ProwJob
-    singular: prowjob
-    plural: prowjobs
-  scope: Namespaced
-  validation:
-    openAPIV3Schema:
-      properties:
-        spec:
-          properties:
-            max_concurrency:
-              type: integer
-              minimum: 0
-            type:
-              type: string
-              enum:
-              - "presubmit"
-              - "postsubmit"
-              - "periodic"
-              - "batch"
-        status:
-          properties:
-            state:
-              type: string
-              enum:
-              - "triggered"
-              - "pending"
-              - "success"
-              - "failure"
-              - "aborted"
-              - "error"
-          anyOf:
-          - not:
-              properties:
-                state:
-                  type: string
-                  enum:
-                  - "success"
-                  - "failure"
-                  - "error"
-          - required:
-            - completionTime
-  additionalPrinterColumns:
-  - name: Job
-    type: string
-    description: The name of the job being run.
-    JSONPath: .spec.job
-  - name: BuildId
-    type: string
-    description: The ID of the job being run.
-    JSONPath: .status.build_id
-  - name: Type
-    type: string
-    description: The type of job being run.
-    JSONPath: .spec.type
-  - name: Org
-    type: string
-    description: The org for which the job is running.
-    JSONPath: .spec.refs.org
-  - name: Repo
-    type: string
-    description: The repo for which the job is running.
-    JSONPath: .spec.refs.repo
-  - name: Pulls
-    type: string
-    description: The pulls for which the job is running.
-    JSONPath: ".spec.refs.pulls[*].number"
-  - name: StartTime
-    type: date
-    description: When the job started running.
-    JSONPath: .status.startTime
-  - name: CompletionTime
-    type: date
-    description: When the job finished running.
-    JSONPath: .status.completionTime
-  - name: State
-    description: The state of the job.
-    type: string
-    JSONPath: .status.state
----
 apiVersion: apps/v1
 kind: Deployment
 metadata:

diff --git a/go.mod b/go.mod
@@ -100,6 +100,8 @@ require (
 	mvdan.cc/xurls/v2 v2.0.0
 	sigs.k8s.io/boskos v0.0.0-20200617235605-f289ba6555ba
 	sigs.k8s.io/controller-runtime v0.8.1-0.20210115141952-3c5b3583fd30
+	sigs.k8s.io/controller-tools v0.4.0
+	sigs.k8s.io/structured-merge-diff/v3 v3.0.1-0.20200706213357-43c19bbb7fba // indirect
 	sigs.k8s.io/yaml v1.2.0
 )
 

diff --git a/go.sum b/go.sum
@@ -500,6 +500,9 @@ github.com/gobuffalo/envy v1.6.5/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9k
 github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
 github.com/gobuffalo/envy v1.7.1 h1:OQl5ys5MBea7OGCdvPbBJWRgnhC/fGona6QKfvFeau8=
 github.com/gobuffalo/envy v1.7.1/go.mod h1:FurDp9+EDPE4aIUS3ZLyD+7/9fpx7YRt/ukY6jIHf0w=
+github.com/gobuffalo/flect v0.2.0 h1:EWCvMGGxOjsgwlWaP+f4+Hh6yrrte7JeFL2S6b+0hdM=
+github.com/gobuffalo/flect v0.2.0/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
 github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
@@ -1547,6 +1550,8 @@ golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roY
 golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200601175630-2caf76543d99/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200616195046-dc31b401abb5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2 h1:FD4wDsP+CQUqh2V12OBOt90pLHVToe58P++fUu3ggV4=
 golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200626171337-aa94e735be7f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200701000337-a32c0cb1d5b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
@@ -1850,7 +1855,6 @@ k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAG
 k8s.io/kubectl v0.17.2/go.mod h1:y4rfLV0n6aPmvbRCqZQjvOp3ezxsFgpqL+zF5jH/lxk=
 k8s.io/kubernetes v1.13.0 h1:qTfB+u5M92k2fCCCVP2iuhgwwSOv1EkAkvQY1tQODD8=
 k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
-k8s.io/kubernetes v1.19.2 h1:sEvBYVM1/H5hqejFR10u8ndreYARV3DiTrqi2AY31ok=
 k8s.io/legacy-cloud-providers v0.17.0/go.mod h1:DdzaepJ3RtRy+e5YhNtrCYwlgyK87j/5+Yfp0L9Syp8=
 k8s.io/legacy-cloud-providers v0.17.4/go.mod h1:FikRNoD64ECjkxO36gkDgJeiQWwyZTuBkhu+yxOc1Js=
 k8s.io/metrics v0.17.2/go.mod h1:3TkNHET4ROd+NfzNxkjoVfQ0Ob4iZnaHmSEA4vYpwLw=
@@ -1908,6 +1912,11 @@ sigs.k8s.io/controller-runtime v0.5.0/go.mod h1:REiJzC7Y00U+2YkMbT8wxgrsX5USpXKG
 sigs.k8s.io/controller-runtime v0.5.4/go.mod h1:JZUwSMVbxDupo0lTJSSFP5pimEyxGynROImSsqIOx1A=
 sigs.k8s.io/controller-runtime v0.8.1-0.20210115141952-3c5b3583fd30 h1:o98Jsn2DtE1Prg8mmW9AfGGXSgpG/GhmqcbcVYp03X8=
 sigs.k8s.io/controller-runtime v0.8.1-0.20210115141952-3c5b3583fd30/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU=
+sigs.k8s.io/controller-runtime v0.6.1 h1:LcK2+nk0kmaOnKGN+vBcWHqY5WDJNJNB/c5pW+sU8fc=
+sigs.k8s.io/controller-runtime v0.6.1/go.mod h1:XRYBPdbf5XJu9kpS84VJiZ7h/u1hF3gEORz0efEja7A=
+sigs.k8s.io/controller-tools v0.4.0 h1:9zIdrc6q9RKke8+DnVPVBVZ+cfF9L0TwM01cxNnklYo=
+sigs.k8s.io/controller-tools v0.4.0/go.mod h1:G9rHdZMVlBDocIxGkK3jHLWqcTMNvveypYJwrvYKjWU=
+sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0=
 sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190302045857-e85c7b244fd2/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=

diff --git a/hack/BUILD.bazel b/hack/BUILD.bazel
@@ -64,6 +64,7 @@ sh_test(
         "$(location @io_k8s_code_generator//cmd/informer-gen)",
         "$(location @io_k8s_code_generator//cmd/lister-gen)",
         "$(location @com_github_go_bindata_go_bindata_v3//go-bindata)",
+        "$(location @io_k8s_sigs_controller_tools//cmd/controller-gen)",
     ],
     data = [
         ":boilerplate/boilerplate.generated.go.txt",  # TODO(fejta): use repo-infra version
@@ -77,6 +78,7 @@ sh_test(
         "@io_k8s_code_generator//cmd/deepcopy-gen",
         "@io_k8s_code_generator//cmd/informer-gen",
         "@io_k8s_code_generator//cmd/lister-gen",
+        "@io_k8s_sigs_controller_tools//cmd/controller-gen",
     ],
     tags = ["lint"],
 )
@@ -90,6 +92,7 @@ sh_binary(
         "$(location @io_k8s_code_generator//cmd/informer-gen)",
         "$(location @io_k8s_code_generator//cmd/lister-gen)",
         "$(location @com_github_go_bindata_go_bindata_v3//go-bindata)",
+        "$(location @io_k8s_sigs_controller_tools//cmd/controller-gen)",
     ],
     data = [
         ":boilerplate/boilerplate.generated.go.txt",  # TODO(fejta): use repo-infra version
@@ -101,6 +104,7 @@ sh_binary(
         "@io_k8s_code_generator//cmd/deepcopy-gen",
         "@io_k8s_code_generator//cmd/informer-gen",
         "@io_k8s_code_generator//cmd/lister-gen",
+        "@io_k8s_sigs_controller_tools//cmd/controller-gen",
     ],
 )
 

diff --git a/hack/tools.go b/hack/tools.go
@@ -26,4 +26,5 @@ import (
 	_ "k8s.io/code-generator/cmd/deepcopy-gen"
 	_ "k8s.io/code-generator/cmd/informer-gen"
 	_ "k8s.io/code-generator/cmd/lister-gen"
+	_ "sigs.k8s.io/controller-tools/cmd/controller-gen"
 )
diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh
@@ -36,7 +36,8 @@ deepcopygen=$PWD/$2
 informergen=$PWD/$3
 listergen=$PWD/$4
 go_bindata=$PWD/$5
-do_clean=${6:-}
+controller_gen=$PWD/$6
+do_clean=${7:-}
 
 cleanup() {
   if [[ -n ${fake_gopath:-} ]]; then chmod u+rwx -R $fake_gopath && rm -rf $fake_gopath; fi
@@ -170,6 +171,14 @@ gen-spyglass-bindata(){
   cd -
 }
 
+gen-prowjob-crd(){
+  clean "./config/prow/cluster" "prowjob_customresourcedefinition.yaml"
+  if [[ -z ${HOME:-} ]]; then export HOME=$PWD; fi
+  $controller_gen crd:preserveUnknownFields=false,crdVersions=v1beta1 paths=./prow/apis/prowjobs/v1 output:stdout |sed '/^$/d' >./config/prow/cluster/prowjob_customresourcedefinition.yaml
+  copyfiles "./config/prow/cluster" "prowjob_customresourcedefinition.yaml"
+  unset HOME
+}
+
 export GO111MODULE=off
 ensure-in-gopath
 old=${GOCACHE:-}
@@ -179,11 +188,13 @@ export GO111MODULE=on
 export GOPROXY=https://proxy.golang.org
 export GOSUMDB=sum.golang.org
 "$go_sdk/bin/go" mod vendor
+export PATH=$PATH:$go_sdk/bin
 export GO111MODULE=off
 export GOCACHE=$old
 gen-deepcopy
 gen-client
 gen-lister
 gen-informer
 gen-spyglass-bindata
+gen-prowjob-crd
 export GO111MODULE=on
diff --git a/hack/verify-codegen.sh b/hack/verify-codegen.sh
@@ -37,6 +37,7 @@ TMP_DIFFROOT="${TEST_TMPDIR}/prow"
 
 mkdir -p "${TMP_DIFFROOT}"
 cp -a "${DIFFROOT}"/{apis,client,spyglass} "${TMP_DIFFROOT}"
+cp "${DIFFROOT}/../config/prow/cluster/prowjob_customresourcedefinition.yaml" "${TMP_DIFFROOT}"
 
 clean=yes # bazel test files are read-only, must first delete
 BUILD_WORKSPACE_DIRECTORY="$SCRIPT_ROOT" "$@" "$clean"
@@ -45,6 +46,7 @@ ret=0
 diff -Naupr "${DIFFROOT}/apis" "${TMP_DIFFROOT}/apis" || ret=$?
 diff -Naupr "${DIFFROOT}/client" "${TMP_DIFFROOT}/client" || ret=$?
 diff -Naupr "${DIFFROOT}/spyglass" "${TMP_DIFFROOT}/spyglass" || ret=$?
+diff -Naupr "${DIFFROOT}/../config/prow/cluster/prowjob_customresourcedefinition.yaml" "${TMP_DIFFROOT}/prowjob_customresourcedefinition.yaml" || ret=$?
 cp -a "${TMP_DIFFROOT}"/{apis,client,spyglass} "${DIFFROOT}"
 if [[ ${ret} -eq 0 ]]; then
   echo "${DIFFROOT} up to date."

diff --git a/prow/getting_started_deploy.md b/prow/getting_started_deploy.md
@@ -160,9 +160,9 @@ Regardless of which object storage you choose, the below adjustments are always
 
 ### Add the prow components to the cluster
 
-Apply the manifest you edited above by executing one of the following two commands:
-
+Apply the crd and the manifest you edited above by executing one of the following two commands:
 
+* `kubectl apply -f config/prow/cluster/prowjob_customresourcedefinition.yaml`
 * `kubectl apply -f config/prow/cluster/starter-s3.yaml`
 * `kubectl apply -f config/prow/cluster/starter-gcs.yaml`