From 108a45bdb103d65d847aa147d8a1c54cbc38285d Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Thu, 24 Jan 2019 04:22:28 -0800 Subject: [PATCH 1/2] data/bootstrap: Replace openshift.sh with cluster-bootstrap With this commit, I take advantage of openshift/cluster-bootstrap@fc5e0941 (start: wire the library-go dynamic client create, 2019-02-05, openshift/cluster-bootstrap#14) to replace our previous openshift.sh (with a minor change to the manifest directory). I'm currently using a cp in bootkube.sh to shift those manifests into the generic directory; I plan on consolidating Openshift into Manifests in pkg/asset/manifests in follow-up work. This change is especially important since the pivot to loopback kubeconfigs in openshift.sh: 82d81d9f53 (data/data/bootstrap: use loopback kubeconfig for API access, 2019-07-24, #2086), because once cluster-bootstrap (launched from bootkube.sh) decides it's done it tears down the bootstrap control plane. Without the bootstrap control plane, further attempts by openshift.sh to push manifests via the loopback kubeconfig fail [1]. We could roll reporting into bootkube.sh as well (dropping progress.service), but Abhinav wanted to keep it separate [2]. [1]: https://bugzilla.redhat.com/show_bug.cgi?id=1748452 [2]: https://github.com/openshift/installer/pull/1381#issuecomment-531878733 --- .../files/usr/local/bin/bootkube.sh.template | 7 ++++ .../files/usr/local/bin/installer-gather.sh | 2 +- .../files/usr/local/bin/openshift.sh | 42 ------------------- .../files/usr/local/bin/report-progress.sh | 1 - .../files/usr/local/bin/report-progress.sh | 1 - .../bootstrap/systemd/units/openshift.service | 12 ------ .../bootstrap/systemd/units/progress.service | 4 +- 7 files changed, 10 insertions(+), 59 deletions(-) delete mode 100755 data/data/bootstrap/files/usr/local/bin/openshift.sh delete mode 100644 data/data/bootstrap/systemd/units/openshift.service diff --git a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template index 561cf767f41..bba3e9dddc7 100755 --- a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template +++ b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template @@ -48,6 +48,13 @@ systemctl restart cri-o.service mkdir --parents ./{bootstrap-manifests,manifests} +if [ ! -f openshift-manifests.done ] +then + echo "Moving OpenShift manifests in with the rest of them" + cp openshift/* manifests/ + touch openshift-manifests.done +fi + if [ ! -f cvo-bootstrap.done ] then echo "Rendering Cluster Version Operator Manifests..." diff --git a/data/data/bootstrap/files/usr/local/bin/installer-gather.sh b/data/data/bootstrap/files/usr/local/bin/installer-gather.sh index 8a3f55ff20b..7f0a1dc7660 100755 --- a/data/data/bootstrap/files/usr/local/bin/installer-gather.sh +++ b/data/data/bootstrap/files/usr/local/bin/installer-gather.sh @@ -4,7 +4,7 @@ ARTIFACTS="/tmp/artifacts" echo "Gathering bootstrap journals ..." mkdir -p "${ARTIFACTS}/bootstrap/journals" -for service in release-image bootkube openshift kubelet crio approve-csr +for service in release-image bootkube kubelet crio approve-csr do journalctl --boot --no-pager --output=short --unit="${service}" > "${ARTIFACTS}/bootstrap/journals/${service}.log" done diff --git a/data/data/bootstrap/files/usr/local/bin/openshift.sh b/data/data/bootstrap/files/usr/local/bin/openshift.sh deleted file mode 100755 index 7e3ce554205..00000000000 --- a/data/data/bootstrap/files/usr/local/bin/openshift.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -set -e - -KUBECONFIG="$1" - -kubectl() { - echo "Executing kubectl $*" >&2 - while true - do - set +e - out=$(oc --config="$KUBECONFIG" "$@" 2>&1) - status=$? - set -e - - if grep --quiet "AlreadyExists" <<< "$out" - then - echo "$out, skipping" >&2 - return - fi - - echo "$out" - if [ "$status" -eq 0 ] - then - return - fi - - echo "kubectl $* failed. Retrying in 5 seconds..." >&2 - sleep 5 - done -} - -for file in $(find . -maxdepth 1 -type f | sort) -do - echo "Creating object from file: $file ..." - kubectl create --filename "$file" - echo "Done creating object from file: $file ..." -done - -# Workaround for https://github.com/opencontainers/runc/pull/1807 -touch /opt/openshift/.openshift.done - -echo "OpenShift installation is done" diff --git a/data/data/bootstrap/files/usr/local/bin/report-progress.sh b/data/data/bootstrap/files/usr/local/bin/report-progress.sh index b305f9590bc..4c60fad2446 100755 --- a/data/data/bootstrap/files/usr/local/bin/report-progress.sh +++ b/data/data/bootstrap/files/usr/local/bin/report-progress.sh @@ -11,7 +11,6 @@ wait_for_existance() { echo "Waiting for bootstrap to complete..." wait_for_existance /opt/openshift/.bootkube.done -wait_for_existance /opt/openshift/.openshift.done echo "Reporting install progress..." while ! oc --config="$KUBECONFIG" create -f - <<-EOF diff --git a/data/data/bootstrap/gcp/files/usr/local/bin/report-progress.sh b/data/data/bootstrap/gcp/files/usr/local/bin/report-progress.sh index 7e0943b7f5d..98ce77bb1ef 100755 --- a/data/data/bootstrap/gcp/files/usr/local/bin/report-progress.sh +++ b/data/data/bootstrap/gcp/files/usr/local/bin/report-progress.sh @@ -11,7 +11,6 @@ wait_for_existance() { echo "Waiting for bootstrap to complete..." wait_for_existance /opt/openshift/.bootkube.done -wait_for_existance /opt/openshift/.openshift.done ## remove the routes setup so that we can open up the blackhole systemctl stop gcp-routes.service diff --git a/data/data/bootstrap/systemd/units/openshift.service b/data/data/bootstrap/systemd/units/openshift.service deleted file mode 100644 index bb0d2207e8d..00000000000 --- a/data/data/bootstrap/systemd/units/openshift.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Bootstrap an OpenShift cluster -Wants=bootkube.service -After=bootkube.service -ConditionPathExists=!/opt/openshift/.openshift.done - -[Service] -WorkingDirectory=/opt/openshift/openshift -ExecStart=/usr/local/bin/openshift.sh /opt/openshift/auth/kubeconfig-loopback - -Restart=on-failure -RestartSec=5s diff --git a/data/data/bootstrap/systemd/units/progress.service b/data/data/bootstrap/systemd/units/progress.service index 09677d09898..81c1350a706 100644 --- a/data/data/bootstrap/systemd/units/progress.service +++ b/data/data/bootstrap/systemd/units/progress.service @@ -1,8 +1,8 @@ [Unit] Description=Report the completion of the cluster bootstrap process # Workaround for https://github.com/systemd/systemd/issues/1312 -Wants=bootkube.service openshift.service -After=bootkube.service openshift.service +Wants=bootkube.service +After=bootkube.service [Service] ExecStart=/usr/local/bin/report-progress.sh /opt/openshift/auth/kubeconfig From b982886ea4b9ef5524b58b7dd079487661fd825f Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Wed, 11 Sep 2019 12:25:56 -0700 Subject: [PATCH 2/2] pkg/asset/machines/userdata: Convert from List to single Secret This may have been what was giving us errors like: Sep 10 19:41:16 ip-10-0-3-146 bootkube.sh[1605]: "99_openshift-cluster-api_master-user-data-secret.yaml": unable to convert asset "99_openshift-cluster-api_master-user-data-secret.yaml" to unstructed Sep 10 19:41:16 ip-10-0-3-146 bootkube.sh[1605]: "99_openshift-cluster-api_worker-user-data-secret.yaml": unable to convert asset "99_openshift-cluster-api_worker-user-data-secret.yaml" to unstructed after failing this cast to *unstructured.Unstructured [2]. I don't have a theory for why the unstructured cast fails for Lists, but that's what I see in local testing: $ cat test.go package main import ( "io/ioutil" "log" "os" "github.com/ghodss/yaml" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" ) func main() { file, err := os.Open("test.yaml") if err != nil { log.Fatal(err) } manifestBytes, err := ioutil.ReadAll(file) if err != nil { log.Fatal(err) } manifestJSON, err := yaml.YAMLToJSON(manifestBytes) if err != nil { log.Fatal(err) } manifestObj, err := runtime.Decode(unstructured.UnstructuredJSONScheme, manifestJSON) if err != nil { log.Fatal(err) } log.Print(manifestObj) log.Print("") manifestUnstructured, ok := manifestObj.(*unstructured.Unstructured) if !ok { log.Fatal(manifestUnstructured) } log.Print(manifestUnstructured) } $ cat <test.yaml > apiVersion: v1 > kind: Secret > metadata: > name: master-user-data > namespace: openshift-machine-api > type: Opaque > data: > disableTemplating: dHJ1ZQo= > userData: dHJ1ZQo= > EOF $ go run test.go 2019/09/12 10:32:07 &{map[kind:Secret metadata:map[name:master-user-data namespace:openshift-machine-api] type:Opaque apiVersion:v1 data:map[userData:dHJ1ZQo= disableTemplating:dHJ1ZQo=]]} 2019/09/12 10:32:07 2019/09/12 10:32:07 &{map[apiVersion:v1 data:map[disableTemplating:dHJ1ZQo= userData:dHJ1ZQo=] kind:Secret metadata:map[name:master-user-data namespace:openshift-machine-api] type:Opaque]} $ cat <test.yaml > kind: List > apiVersion: v1 > metadata: > resourceVersion: "" > selfLink: "" > items: > - apiVersion: v1 > kind: Secret > metadata: > name: master-user-data > namespace: openshift-machine-api > type: Opaque > data: > disableTemplating: dHJ1ZQo= > userData: dHJ1ZQo= > EOF $ go run test.go 2019/09/12 10:33:20 &{map[kind:List metadata:map[resourceVersion: selfLink:] apiVersion:v1] [{map[metadata:map[name:master-user-data namespace:openshift-machine-api] type:Opaque apiVersion:v1 data:map[disableTemplating:dHJ1ZQo= userData:dHJ1ZQo=] kind:Secret]}]} 2019/09/12 10:33:20 2019/09/12 10:33:20 exit status 1 [1]: https://github.com/openshift/installer/pull/1381#issuecomment-530366997 [2]: https://github.com/openshift/library-go/blob/7d4acc018c610623ee1413b1e7aebe2ac675d35f/pkg/assets/create/creater.go#L264 --- pkg/asset/machines/master.go | 3 +-- pkg/asset/machines/userdata.go | 37 +++++++++++++--------------------- pkg/asset/machines/worker.go | 3 +-- 3 files changed, 16 insertions(+), 27 deletions(-) diff --git a/pkg/asset/machines/master.go b/pkg/asset/machines/master.go index b396959e51a..d085c422ef7 100644 --- a/pkg/asset/machines/master.go +++ b/pkg/asset/machines/master.go @@ -266,8 +266,7 @@ func (m *Master) Generate(dependencies asset.Parents) error { return fmt.Errorf("invalid Platform") } - userDataMap := map[string][]byte{"master-user-data": mign.File.Data} - data, err := userDataList(userDataMap) + data, err := userDataSecret("master-user-data", mign.File.Data) if err != nil { return errors.Wrap(err, "failed to create user-data secret for master machines") } diff --git a/pkg/asset/machines/userdata.go b/pkg/asset/machines/userdata.go index 9a666b53c69..c3fa6179464 100644 --- a/pkg/asset/machines/userdata.go +++ b/pkg/asset/machines/userdata.go @@ -8,34 +8,25 @@ import ( "github.com/pkg/errors" ) -var userDataListTmpl = template.Must(template.New("user-data-list").Parse(` -kind: List -apiVersion: v1 +var userDataTmpl = template.Must(template.New("user-data").Parse(`apiVersion: v1 +kind: Secret metadata: - resourceVersion: "" - selfLink: "" -items: -{{- range $name, $content := . }} -- apiVersion: v1 - kind: Secret - metadata: - name: {{$name}} - namespace: openshift-machine-api - type: Opaque - data: - disableTemplating: "dHJ1ZQo=" - userData: {{$content}} -{{- end}} + name: {{.name}} + namespace: openshift-machine-api +type: Opaque +data: + disableTemplating: "dHJ1ZQo=" + userData: {{.content}} `)) -func userDataList(data map[string][]byte) ([]byte, error) { - encodedData := map[string]string{} - for name, content := range data { - encodedData[name] = base64.StdEncoding.EncodeToString(content) +func userDataSecret(name string, content []byte) ([]byte, error) { + encodedData := map[string]string{ + "name": name, + "content": base64.StdEncoding.EncodeToString(content), } buf := &bytes.Buffer{} - if err := userDataListTmpl.Execute(buf, encodedData); err != nil { - return nil, errors.Wrap(err, "failed to execute content.UserDataListTmpl") + if err := userDataTmpl.Execute(buf, encodedData); err != nil { + return nil, errors.Wrap(err, "failed to execute user-data template") } return buf.Bytes(), nil } diff --git a/pkg/asset/machines/worker.go b/pkg/asset/machines/worker.go index 39eccdffec0..6344f791397 100644 --- a/pkg/asset/machines/worker.go +++ b/pkg/asset/machines/worker.go @@ -260,8 +260,7 @@ func (w *Worker) Generate(dependencies asset.Parents) error { } } - userDataMap := map[string][]byte{"worker-user-data": wign.File.Data} - data, err := userDataList(userDataMap) + data, err := userDataSecret("worker-user-data", wign.File.Data) if err != nil { return errors.Wrap(err, "failed to create user-data secret for worker machines") }