From 1a6a13d20f835c689fdb6d186a02a7fdab1c5b1f Mon Sep 17 00:00:00 2001 From: Chuck Ha Date: Thu, 9 Nov 2017 17:32:56 -0500 Subject: [PATCH] Quickstart fixes * `jsonnet fmt` our jsonnet files * Bug fixes for recent plugin changes * Fixup the ksonnet that generates the quickstart * Worker configuration is entirely through environment variables Signed-off-by: Chuck Ha --- examples/ksonnet/components/00-rbac.jsonnet | 6 +- .../ksonnet/components/10-configmaps.jsonnet | 368 ++++++++---------- examples/ksonnet/components/20-pod.jsonnet | 7 +- examples/quickstart.yaml | 123 +++--- pkg/worker/config.go | 4 - plugins.d/e2e.tmpl | 4 +- 6 files changed, 219 insertions(+), 293 deletions(-) diff --git a/examples/ksonnet/components/00-rbac.jsonnet b/examples/ksonnet/components/00-rbac.jsonnet index 116ca5d07..21937bd9a 100644 --- a/examples/ksonnet/components/00-rbac.jsonnet +++ b/examples/ksonnet/components/00-rbac.jsonnet @@ -47,12 +47,12 @@ local clusterRoleBinding = crb.new() + crb.mixin.metadata.mixinInstance(conf.metadata) + # TODO: replace with `crb.mixinroleRef.kind("ClusterRole") when https://github.com/ksonnet/ksonnet-lib/issues/53 closes. - {roleRef: {kind: "ClusterRole"}} + + { roleRef: { kind: "ClusterRole" } } + crb.mixin.roleRef.apiGroup("rbac.authorization.k8s.io") + crb.mixin.roleRef.name(conf.serviceAccount.name) + crb.subjects([ # TODO: replace with `crb.subjectsType.kind("ServiceAccount")` when https://github.com/ksonnet/ksonnet-lib/issues/43 closes. - {kind: "ServiceAccount"} + + { kind: "ServiceAccount" } + crb.subjectsType.name(conf.serviceAccount.name) + crb.subjectsType.namespace(conf.namespace), ]); @@ -75,5 +75,5 @@ local optRbacObj = k.core.v1.list.new([ namespace, - serviceaccount + serviceaccount, ] + optRbacObj) diff --git a/examples/ksonnet/components/10-configmaps.jsonnet b/examples/ksonnet/components/10-configmaps.jsonnet index a661b94b1..50b8021d0 100644 --- a/examples/ksonnet/components/10-configmaps.jsonnet +++ b/examples/ksonnet/components/10-configmaps.jsonnet @@ -15,6 +15,8 @@ local k = import "ksonnet.beta.2/k.libsonnet"; local kubecfg = import "kubecfg.libsonnet"; local configMap = k.core.v1.configMap; +local c = k.core.v1.pod.mixin.spec.containersType; +local ds = k.extensions.v1beta1.daemonSet; local conf = { namespace: "heptio-sonobuoy", @@ -65,254 +67,196 @@ local sonobuoyConfigData = { "ServerGroups", "ServiceAccounts", "Services", - "StatefulSets" + "StatefulSets", ], Filters: { LabelSelector: "", - Namespaces: ".*" + Namespaces: ".*", }, Server: { advertiseaddress: "sonobuoy-master:8080", bindaddress: "0.0.0.0", bindport: 8080, - timeoutseconds: 5400 + timeoutseconds: 5400, }, PluginNamespace: "heptio-sonobuoy", Plugins: [ - {name: "systemd_logs"}, - {name: "e2e"} - ] + { name: "systemd_logs" }, + { name: "e2e" }, + ], }; -local systemdlogsConfig = { - name: "systemd_logs", - driver: "DaemonSet", - resultType: "systemd_logs", - spec: { - tolerations: [ - { +local tolerations = [ + { key: "node-role.kubernetes.io/master", operator: "Exists", effect: "NoSchedule", - }, - { + }, + { key: "CriticalAddonsOnly", operator: "Exists", - }, - ], - hostNetwork: true, - hostIPC: true, - hostPID: true, - dnsPolicy: "ClusterFirstWithHostNet", - containers: [ - { + }, +]; + +local systemdconf = { + pluginName: "systemd_logs", + name: "sonobuoy-systemd-logs-config-{{.SessionID}}", + annotations: { + "sonobuoy-plugin": $.pluginName, + "sonobuoy-driver": "DaemonSet", + "sonobuoy-result-type": $.pluginName, + }, + labels: { + component: "sonobuoy", + tier: "analysis", + "sonobuoy-run": "{{.SessionID}}", + }, + namespace: "{{.Namespace}}", + + serviceAccountName: "sonobuoy-serviceaccount", + + rootDir: "/node", + resultsDir: "/tmp/results", + selector: { + "sonobuoy-run": "{{.SessionID}}", + }, + systemd: { name: "systemd-logs", - command: [ - "sh", - "-c", - "/get_systemd_logs.sh && sleep 3600" - ], - env: [ - { - name: "NODE_NAME", - valueFrom: { - fieldRef: { - apiVersion: "v1", - fieldPath: "spec.nodeName", - }, - } - }, - { - name: "RESULTS_DIR", - value: "/tmp/results", - }, - { - name: "CHROOT_DIR", - value: "/node", - }, - ], image: "gcr.io/heptio-images/sonobuoy-plugin-systemd-logs:latest", - imagePullPolicy: "Always", - securityContext: { - privileged: true, - }, - volumeMounts: [ - { - mountPath: "/node", - name: "root", - }, - { - mountPath: "/tmp/results", - name: "results", - }, - { - mountPath: "/etc/sonobuoy", - name: "config", - }, - ], - }, - { + command: ["sh", "-c", "/get_systemd_logs.sh && sleep 3600"], + }, + workerContainer: { name: "sonobuoy-worker", - command: [ - "sh", - "-c", - "/sonobuoy worker single-node -v 5 --logtostderr && sleep 3600", - ], - env: [ - { - name: "NODE_NAME", - valueFrom: { - fieldRef: { - apiVersion: "v1", - fieldPath: "spec.nodeName", - }, - }, - }, - { - name: "RESULTS_DIR", - value: "/tmp/results", - }, - ], image: "gcr.io/heptio-images/sonobuoy:master", - imagePullPolicy: "Always", - securityContext: { - privileged: true, - }, - volumeMounts: [ - { - mountPath: "/tmp/results", - name: "results", - }, - { - mountPath: "/etc/sonobuoy", - name: "config", - }, - ], - }, - ], - volumes: [ - { - name: "root", - hostPath: { - path: "/", - }, - }, - { - name: "results", - emptyDir: {}, - }, - { - name: "config", - configMap: { - name: "__SONOBUOY_CONFIGMAP__", - }, - }, - ], - }, + command: ["sh", "-c", "/sonobuoy worker single-node -v 5 --logtostderr && sleep 3600"], + }, }; -local e2eConfig = { - name: "e2e", - driver: "Job", - resultType: "e2e", - spec: { +local systemdContainer = + c.new(systemdconf.systemd.name, systemdconf.systemd.image) + + c.imagePullPolicy("Always") + + c.command(systemdconf.systemd.command) + + c.env([ + c.envType.fromFieldPath("NODE_NAME", "spec.nodeName"), + c.envType.new("RESULTS_DIR", systemdconf.resultsDir), + c.envType.new("CHROOT_DIR", systemdconf.rootDir), + ]) + + c.mixin.securityContext.privileged(true) + + c.volumeMounts([ + c.volumeMountsType.new("results", systemdconf.resultsDir), + c.volumeMountsType.new("root", systemdconf.rootDir), + ]); + +local systemdWorker = + c.new(systemdconf.workerContainer.name, systemdconf.workerContainer.image) + + c.imagePullPolicy("Always") + + c.command(systemdconf.workerContainer.command) + + c.env([ + c.envType.fromFieldPath("NODE_NAME", "spec.nodeName"), + c.envType.new("RESULTS_DIR", systemdconf.resultsDir), + c.envType.new("MASTER_URL", "{{.MasterAddress}}"), + c.envType.new("RESULT_TYPE", systemdconf.pluginName), + ]) + + c.volumeMounts([ + c.volumeMountsType.new("results", systemdconf.resultsDir), + ]); + +local systemdDaemonSet = + ds.new() + + ds.mixin.metadata.name(systemdconf.name) + + ds.mixin.metadata.annotations(systemdconf.annotations) + + ds.mixin.metadata.labels(systemdconf.labels) + + ds.mixin.metadata.namespace(systemdconf.namespace) + + ds.mixin.spec.selector.matchLabels(systemdconf.selector) + + ds.mixin.spec.template.metadata.labels(systemdconf.labels) + + ds.mixin.spec.template.spec.containers([systemdContainer, systemdWorker]) + + ds.mixin.spec.template.spec.volumes([ + ds.mixin.spec.template.spec.volumesType.fromEmptyDir("results"), + ds.mixin.spec.template.spec.volumesType.fromHostPath("root", "/"), + ]); + + +local pluginconf = { + pluginName: "e2e", + name: "sonobuoy-e2e-job-{{.SessionID}}", + annotations: { + "sonobuoy-plugin": $.pluginName, + "sonobuoy-driver": "Job", + "sonobuoy-result-type": $.pluginName, + }, + labels: { + component: "sonobuoy", + tier: "analysis", + "sonobuoy-run": "{{.SessionID}}", + }, + namespace: "{{.Namespace}}", + serviceAccountName: "sonobuoy-serviceaccount", - tolerations: [ - { - key: "node-role.kubernetes.io/master", - operator: "Exists", - effect: "NoSchedule", - }, - { - key: "CriticalAddonsOnly", - operator: "Exists", - }, - ], - restartPolicy: "Never", - containers: [ - { - name: "e2e", - image: "gcr.io/heptio-images/kube-conformance:v1.8", - imagePullPolicy: "Always", - # NOTE: Full conformance can take a while depending on your cluster size. - # As a result, only a single test is set atm to verify correctness. - # Operators that want the complete test results can comment out the - # env section. - env: [ - { - name: "E2E_FOCUS", - value: "Pods should be submitted and removed", - }, - ], - volumeMounts: [ - { - name: "results", - mountPath: "/tmp/results", - }, - ], - }, - { + + resultsDir: "/tmp/results", + + e2ec: { + name: $.pluginName, + image: "gcr.io/heptio-images/kube-conformance:latest", + }, + workerContainer: { name: "sonobuoy-worker", - command: [ - "sh", - "-c", - "/sonobuoy worker global -v 5 --logtostderr", - ], - env: [ - { - name: "NODE_NAME", - valueFrom: { - fieldRef: { - apiVersion: "v1", - fieldPath: "spec.nodeName", - }, - }, - }, - { - name: "RESULTS_DIR", - value: "/tmp/results", - }, - ], image: "gcr.io/heptio-images/sonobuoy:master", - imagePullPolicy: "Always", - volumeMounts: [ - { - name: "config", - mountPath: "/etc/sonobuoy", - }, - { - name: "results", - mountPath: "/tmp/results", - }, - ], - }, - ], - volumes: [ - { - name: "results", - emptyDir: {}, - }, - { - name: "config", - configMap: { - # This will be rewritten when the JobPlugin driver goes to launch the pod. - name: "__SONOBUOY_CONFIGMAP__", - }, - }, - ], - }, + command: ["sh", "-c", "/sonobuoy worker global -v 5 --logtostderr"], + }, }; +local e2eContainer = + c.new(pluginconf.e2ec.name, pluginconf.e2ec.image) + + c.imagePullPolicy("Always") + + c.env([ + c.envType.new("E2E_FOCUS", "Pods should be submitted and removed"), + ]) + + c.volumeMounts([ + c.volumeMountsType.new("results", pluginconf.resultsDir), + ]); + +local e2eWorker = + c.new(pluginconf.workerContainer.name, pluginconf.workerContainer.image) + + c.imagePullPolicy("Always") + + c.command(pluginconf.workerContainer.command) + + c.env([ + c.envType.fromFieldPath("NODE_NAME", "spec.nodeName"), + c.envType.new("RESULTS_DIR", pluginconf.resultsDir), + c.envType.new("MASTER_URL", "{{.MasterAddress}}"), + c.envType.new("RESULT_TYPE", pluginconf.pluginName), + ]) + + c.volumeMounts([ + c.volumeMountsType.new("results", pluginconf.resultsDir), + ]); + +local e2epod = + local p = k.core.v1.pod; + p.new() + + // Metaddata + p.mixin.metadata.name(pluginconf.pluginName) + + p.mixin.metadata.annotations(pluginconf.annotations) + + p.mixin.metadata.labels(pluginconf.labels) + + p.mixin.metadata.namespace(pluginconf.namespace) + + // Spec + p.mixin.spec.serviceAccountName(pluginconf.serviceAccountName) + + p.mixin.spec.tolerations(tolerations) + + p.mixin.spec.restartPolicy("Never") + + p.mixin.spec.containers([e2eContainer, e2eWorker]) + + p.mixin.spec.volumes([ + p.mixin.spec.volumesType.fromEmptyDir("results"), + ]); + local plugins = { - "systemdlogs.yaml": kubecfg.manifestYaml(systemdlogsConfig), - "e2e.yaml": kubecfg.manifestYaml(e2eConfig), + "systemd_logs.tmpl": kubecfg.manifestYaml(systemdDaemonSet), + "e2e.tmpl": kubecfg.manifestYaml(e2epod), }; local sonobuoyConfig = configMap.new() + configMap.mixin.metadata.name(conf.sonobuoyCfg.name) + configMap.mixin.metadata.namespace(conf.namespace) + configMap.mixin.metadata.labels(conf.labels) + - configMap.data({"config.json": kubecfg.manifestJson(sonobuoyConfigData)}); + configMap.data({ "config.json": kubecfg.manifestJson(sonobuoyConfigData) }); local pluginConfigs = configMap.new() + configMap.mixin.metadata.name(conf.pluginsCfg.name) + diff --git a/examples/ksonnet/components/20-pod.jsonnet b/examples/ksonnet/components/20-pod.jsonnet index ce2127c73..a91212f41 100644 --- a/examples/ksonnet/components/20-pod.jsonnet +++ b/examples/ksonnet/components/20-pod.jsonnet @@ -50,7 +50,7 @@ local conf(opts) = { "/sonobuoy master --no-exit=true -v ", if opts.debug then "5" else "3", " --logtostderr", - if opts.debug then " --debug" else "" + if opts.debug then " --debug" else "", ]), ], imagePullPolicy: opts.pullPolicy, @@ -88,7 +88,7 @@ local conf(opts) = { { - objects(pullPolicy="Always", debug=false) :: + objects(pullPolicy="Always", debug=false):: local opts = { pullPolicy: pullPolicy, debug: debug, @@ -118,6 +118,5 @@ local conf(opts) = { svc.mixin.metadata.labels(myconf.labels) + svc.mixin.spec.type(myconf.service.type); - k.core.v1.list.new([sonobuoyPod, sonobuoyService]) + k.core.v1.list.new([sonobuoyPod, sonobuoyService]), } - diff --git a/examples/quickstart.yaml b/examples/quickstart.yaml index d568d97ab..b08b533e8 100644 --- a/examples/quickstart.yaml +++ b/examples/quickstart.yaml @@ -116,41 +116,29 @@ data: apiVersion: v1 kind: Pod metadata: - name: "sonobuoy-e2e-job-{{.SessionID}}" annotations: - "sonobuoy-plugin": "e2e" - "sonobuoy-driver": "Job" - "sonobuoy-result-type": "e2e" + sonobuoy-driver: Job + sonobuoy-plugin: e2e + sonobuoy-result-type: e2e labels: component: sonobuoy - tier: analysis - "sonobuoy-run": "{{.SessionID}}" - namespace: "{{.Namespace}}" + sonobuoy-run: '{{.SessionID}}' + tier: analysis + name: e2e + namespace: '{{.Namespace}}' spec: - serviceAccountName: sonobuoy-serviceaccount - tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule - - key: CriticalAddonsOnly - operator: Exists - restartPolicy: Never containers: - - name: e2e + - env: + - name: E2E_FOCUS + value: Pods should be submitted and removed image: gcr.io/heptio-images/kube-conformance:latest imagePullPolicy: Always - # NOTE: Full conformance can take a while depending on your cluster size. - # As a result, only a single test is set atm to verify correctness. - # Operators that want the complete test results can comment out the - # env section. - env: - - name: E2E_FOCUS - value: "Pods should be submitted and removed" + name: e2e volumeMounts: - - name: results - mountPath: /tmp/results - - name: sonobuoy-worker - command: + - mountPath: /tmp/results + name: results + readOnly: false + - command: - sh - -c - /sonobuoy worker global -v 5 --logtostderr @@ -158,60 +146,58 @@ data: - name: NODE_NAME valueFrom: fieldRef: - apiVersion: v1 fieldPath: spec.nodeName - name: RESULTS_DIR value: /tmp/results - name: MASTER_URL - value: "{{.MasterAddress}}" + value: '{{.MasterAddress}}' - name: RESULT_TYPE - value: "e2e" + value: e2e image: gcr.io/heptio-images/sonobuoy:master imagePullPolicy: Always + name: sonobuoy-worker volumeMounts: - - name: results - mountPath: /tmp/results + - mountPath: /tmp/results + name: results + readOnly: false + restartPolicy: Never + serviceAccountName: sonobuoy-serviceaccount + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - key: CriticalAddonsOnly + operator: Exists volumes: - - name: results - emptyDir: {} + - emptyDir: {} + name: results systemd_logs.tmpl: | apiVersion: extensions/v1beta1 kind: DaemonSet metadata: - name: "sonobuoy-systemd-logs-config-{{.SessionID}}" annotations: - "sonobuoy-plugin": "systemd_logs" - "sonobuoy-driver": "DaemonSet" - "sonobuoy-result-type": "systemd_logs" + sonobuoy-driver: DaemonSet + sonobuoy-plugin: systemd_logs + sonobuoy-result-type: systemd_logs labels: component: sonobuoy + sonobuoy-run: '{{.SessionID}}' tier: analysis - "sonobuoy-run": "{{.SessionID}}" - namespace: {{.Namespace}} + name: sonobuoy-systemd-logs-config-{{.SessionID}} + namespace: '{{.Namespace}}' spec: selector: matchLabels: - "sonobuoy-run": "{{.SessionID}}" + sonobuoy-run: '{{.SessionID}}' template: metadata: labels: component: sonobuoy + sonobuoy-run: '{{.SessionID}}' tier: analysis - "sonobuoy-run": "{{.SessionID}}" spec: - tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule - - key: CriticalAddonsOnly - operator: Exists - hostNetwork: true - hostIPC: true - hostPID: true - dnsPolicy: ClusterFirstWithHostNet containers: - - name: systemd-logs - command: + - command: - sh - -c - /get_systemd_logs.sh && sleep 3600 @@ -219,7 +205,6 @@ data: - name: NODE_NAME valueFrom: fieldRef: - apiVersion: v1 fieldPath: spec.nodeName - name: RESULTS_DIR value: /tmp/results @@ -227,15 +212,17 @@ data: value: /node image: gcr.io/heptio-images/sonobuoy-plugin-systemd-logs:latest imagePullPolicy: Always + name: systemd-logs securityContext: privileged: true volumeMounts: - - mountPath: /node - name: root - mountPath: /tmp/results name: results - - name: sonobuoy-worker - command: + readOnly: false + - mountPath: /node + name: root + readOnly: false + - command: - sh - -c - /sonobuoy worker single-node -v 5 --logtostderr && sleep 3600 @@ -243,26 +230,26 @@ data: - name: NODE_NAME valueFrom: fieldRef: - apiVersion: v1 fieldPath: spec.nodeName - name: RESULTS_DIR value: /tmp/results - name: MASTER_URL - value: "{{.MasterAddress}}" + value: '{{.MasterAddress}}' + - name: RESULT_TYPE + value: systemd_logs image: gcr.io/heptio-images/sonobuoy:master imagePullPolicy: Always - securityContext: - privileged: true + name: sonobuoy-worker volumeMounts: - mountPath: /tmp/results name: results + readOnly: false volumes: - - name: root - hostPath: + - emptyDir: {} + name: results + - hostPath: path: / - - name: results - emptyDir: {} - + name: root kind: ConfigMap metadata: labels: diff --git a/pkg/worker/config.go b/pkg/worker/config.go index 54b19feac..6b895d806 100644 --- a/pkg/worker/config.go +++ b/pkg/worker/config.go @@ -51,10 +51,6 @@ func LoadConfig() (*plugin.WorkerConfig, error) { setConfigDefaults(config) - if err = viper.ReadInConfig(); err != nil { - return nil, errors.WithStack(err) - } - if err = viper.Unmarshal(config); err != nil { return nil, errors.WithStack(err) } diff --git a/plugins.d/e2e.tmpl b/plugins.d/e2e.tmpl index 238831bd6..be9b1ff4b 100644 --- a/plugins.d/e2e.tmpl +++ b/plugins.d/e2e.tmpl @@ -7,8 +7,8 @@ metadata: "sonobuoy-driver": "Job" labels: component: sonobuoy - tier: analysis - "sonobuoy-run": "{{.SessionID}}" + tier: analysis + "sonobuoy-run": "{{.SessionID}}" namespace: "{{.Namespace}}" spec: serviceAccountName: sonobuoy-serviceaccount