From 5177e7476a61d532c9be0b1bb6ad1f21c3981e3a Mon Sep 17 00:00:00 2001 From: David Eads Date: Tue, 7 May 2019 14:16:09 -0400 Subject: [PATCH 1/8] gather: suffix container logs with container ids for uniqueness --- data/data/bootstrap/files/usr/local/bin/installer-gather.sh | 4 ++-- .../bootstrap/files/usr/local/bin/installer-masters-gather.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/data/data/bootstrap/files/usr/local/bin/installer-gather.sh b/data/data/bootstrap/files/usr/local/bin/installer-gather.sh index 65b6c832d..edc2d9d92 100755 --- a/data/data/bootstrap/files/usr/local/bin/installer-gather.sh +++ b/data/data/bootstrap/files/usr/local/bin/installer-gather.sh @@ -14,8 +14,8 @@ mkdir -p "${ARTIFACTS}/bootstrap/containers" sudo crictl ps --all --quiet | while read -r container do container_name="$(sudo crictl ps -a --id "${container}" -v | grep -oP "Name: \\K(.*)")" - sudo crictl logs "${container}" >& "${ARTIFACTS}/bootstrap/containers/${container_name}.log" - sudo crictl inspect "${container}" >& "${ARTIFACTS}/bootstrap/containers/${container_name}.inspect" + sudo crictl logs "${container}" >& "${ARTIFACTS}/bootstrap/containers/${container_name}-${container}.log" + sudo crictl inspect "${container}" >& "${ARTIFACTS}/bootstrap/containers/${container_name}-${container}.inspect" done mkdir -p "${ARTIFACTS}/bootstrap/pods" sudo podman ps --all --quiet | while read -r container diff --git a/data/data/bootstrap/files/usr/local/bin/installer-masters-gather.sh b/data/data/bootstrap/files/usr/local/bin/installer-masters-gather.sh index 5f885535b..4fd080ca0 100755 --- a/data/data/bootstrap/files/usr/local/bin/installer-masters-gather.sh +++ b/data/data/bootstrap/files/usr/local/bin/installer-masters-gather.sh @@ -15,8 +15,8 @@ mkdir -p "${ARTIFACTS}/containers" for container in $(crictl ps --all --quiet) do container_name=$(crictl ps -a --id "${container}" -v | grep -oP "Name: \\K(.*)") - crictl logs "${container}" >& "${ARTIFACTS}/containers/${container_name}.log" - crictl inspect "${container}" >& "${ARTIFACTS}/containers/${container_name}.inspect" + crictl logs "${container}" >& "${ARTIFACTS}/containers/${container_name}-${container}.log" + crictl inspect "${container}" >& "${ARTIFACTS}/containers/${container_name}-${container}.inspect" done for container in $(podman ps --all --quiet) do From f52fd4e1b238f88f4c2ea0921a08008908dd02c1 Mon Sep 17 00:00:00 2001 From: Jeremiah Stuever Date: Thu, 9 May 2019 15:03:19 -0400 Subject: [PATCH 2/8] bootstrap/files: installer-gather.sh to gather more rendered-assets This change adds sudo when gathering rendered-assets within /usr/local/bin/installer-gather.sh. It also changes the owner and permissions on the copied files so that the core user has access to read them. In addition, the method used to retrieve assets from the control-plane nodes has been modified to make it less noisy on error. --- .../bootstrap/files/usr/local/bin/installer-gather.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/data/data/bootstrap/files/usr/local/bin/installer-gather.sh b/data/data/bootstrap/files/usr/local/bin/installer-gather.sh index 21a5a7dfe..5b0aed48c 100755 --- a/data/data/bootstrap/files/usr/local/bin/installer-gather.sh +++ b/data/data/bootstrap/files/usr/local/bin/installer-gather.sh @@ -26,7 +26,9 @@ done echo "Gathering rendered assets..." mkdir -p "${ARTIFACTS}/rendered-assets" -cp -r /var/opt/openshift/ "${ARTIFACTS}/rendered-assets" +sudo cp -r /var/opt/openshift/ "${ARTIFACTS}/rendered-assets" +sudo chown -R "${USER}":"${USER}" "${ARTIFACTS}/rendered-assets" +sudo find "${ARTIFACTS}/rendered-assets" -type d -print0 | xargs -0 sudo chmod u+x # remove sensitive information # TODO leave tls.crt inside of secret yaml files find "${ARTIFACTS}/rendered-assets" -name "*secret*" -print0 | xargs -0 rm @@ -107,10 +109,10 @@ mapfile -t MASTERS < "${ARTIFACTS}/resources/masters.list" for master in "${MASTERS[@]}" do echo "Collecting info from ${master}" - scp -o PreferredAuthentications=publickey -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null /usr/local/bin/installer-masters-gather.sh "core@${master}:" + scp -o PreferredAuthentications=publickey -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null -q /usr/local/bin/installer-masters-gather.sh "core@${master}:" mkdir -p "${ARTIFACTS}/control-plane/${master}" ssh -o PreferredAuthentications=publickey -o StrictHostKeyChecking=false -o UserKnownHostsFile=/dev/null "core@${master}" -C 'sudo ./installer-masters-gather.sh' ~/log-bundle.tar.gz echo "Log bundle written to ~/log-bundle.tar.gz" From 48b21ac9f841d59de51383c5265a61bcea32313d Mon Sep 17 00:00:00 2001 From: trown Date: Thu, 9 May 2019 16:03:43 -0400 Subject: [PATCH 3/8] openstack: fix comments from 1733 --- images/openstack/Dockerfile.ci | 3 ++- images/openstack/rdo-stein.gpg | 20 ++++++++++++++++++++ images/openstack/rdo-stein.repo | 3 ++- 3 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 images/openstack/rdo-stein.gpg diff --git a/images/openstack/Dockerfile.ci b/images/openstack/Dockerfile.ci index 58ad0543f..a26e9b0c8 100644 --- a/images/openstack/Dockerfile.ci +++ b/images/openstack/Dockerfile.ci @@ -7,7 +7,8 @@ RUN hack/build.sh FROM registry.svc.ci.openshift.org/origin/4.1:base COPY --from=builder /go/src/github.com/openshift/installer/bin/openshift-install /bin/openshift-install -COPY --from=builder images/openstack/rdo-stein.repo /etc/yum.repos.d/rdo-stein.repo +COPY --from=builder /go/src/github.com/openshift/installer/images/openstack/rdo-stein.repo /etc/yum.repos.d/rdo-stein.repo +COPY --from=builder /go/src/github.com/openshift/installer/images/openstack/rdo-stein.gpg /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud COPY --from=registry.svc.ci.openshift.org/openshift/origin-v4.0:cli /usr/bin/oc /bin/oc RUN yum install --setopt=tsflags=nodocs -y \ diff --git a/images/openstack/rdo-stein.gpg b/images/openstack/rdo-stein.gpg new file mode 100644 index 000000000..91f8e1c52 --- /dev/null +++ b/images/openstack/rdo-stein.gpg @@ -0,0 +1,20 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2.0.22 (GNU/Linux) + +mQENBFVWcCcBCACfm3eQ0526/I0/p7HpR0NjK7K307XHhnbcbZv1sDUjQABDaqh0 +N4gnZcovf+3fj6pcdOmeOpGI0cKE7Fh68RbEIqyjB7l7+j1grjewR0oCFFZ38KGm +j+DWQrj1IJW7JU5fH/G0Cu66ix+dJPcuTB3PJTqXN3ce+4TuG09D+epgwfbHlqaT +pH2qHCu2uiGj/AaRSM/ZZzcInMaeleHSB+NChvaQ0W/m+kK5d/20d7sfkaTfI/pY +SrodCfVTYxfKAd0TLW03kimHs5/Rdz+iZWecVKv6aFxzaywbrOjmOsy2q0kEWIwX +MTZrq6cBRRuWyiXsI2zT2YHQ4UK44IxINiaJABEBAAG0WkNlbnRPUyBDbG91ZCBT +SUcgKGh0dHA6Ly93aWtpLmNlbnRvcy5vcmcvU3BlY2lhbEludGVyZXN0R3JvdXAv +Q2xvdWQpIDxzZWN1cml0eUBjZW50b3Mub3JnPokBOQQTAQIAIwUCVVZwJwIbAwcL +CQgHAwIBBhUIAgkKCwQWAgMBAh4BAheAAAoJEPm5/ud2RCnmATUH/3HDtWxpFkmy +FiA3VGkMt5dp3bgCRSd84X6Orfx1LARowpI4LomCGglGBGXVJePBacwcclorbLaz +uWrW/wU0efz0aDB5c4NPg/yXfNvujvlda8ADJwZXVBQphzvaIKwl4PqBsEnxC10I +93T/0iyphAhfMRJ5R8AbEHMj7uF+TWTX/JoyQagllMqWTwoP4DFRutPdOmmjwvSV +kWItH7hq6z9+M4dhlqeoOvPbL5oCxX7TVmLck02Q5gI4syULOa7sqntzUQKFkhWp +9U0+5KrBQBKezrurrrkq/WZR3WNE1KQfNQ77f7S2JcXJdOaKgJ7xe7Y2flPq98Aq +wKXK7l1c3dc= +=W6yF +-----END PGP PUBLIC KEY BLOCK----- diff --git a/images/openstack/rdo-stein.repo b/images/openstack/rdo-stein.repo index 758e884ec..4edae13f0 100644 --- a/images/openstack/rdo-stein.repo +++ b/images/openstack/rdo-stein.repo @@ -2,5 +2,6 @@ name=OpenStack Stein Repository baseurl=http://mirror.centos.org/centos/7/cloud/$basearch/openstack-stein/ #mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=cloud-openstack-stein -gpgcheck=0 +gpgcheck=1 enabled=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud From 2eabc7d2b4b86b8b18b6d59cd09df348b10cd6ac Mon Sep 17 00:00:00 2001 From: DanyC97 Date: Fri, 10 May 2019 10:54:28 +0100 Subject: [PATCH 4/8] Fix the UPI's Readme deploy-packet url --- upi/metal/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upi/metal/README.md b/upi/metal/README.md index 1af0e1ca3..2701d394a 100644 --- a/upi/metal/README.md +++ b/upi/metal/README.md @@ -16,7 +16,7 @@ Setup `default` AWS cli profile on the host that will run the example terraform ### Packet -Setup a Project in Packet.net that will be used to deploy servers, for example using this [guide](packet-deploy-server) +Setup a Project in Packet.net that will be used to deploy servers, for example using this [guide][packet-deploy-server] Setup API keys for your Project in Packet.net using this [guide][packet-api-keys] From b769e8a66da4cf07c198bd405a669f06b6ab0e09 Mon Sep 17 00:00:00 2001 From: Emilio Garcia Date: Tue, 7 May 2019 16:45:56 -0400 Subject: [PATCH 5/8] deprovision more robust --- .../openstack/openstack_deprovision.go | 56 +++++++++++-------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/pkg/destroy/openstack/openstack_deprovision.go b/pkg/destroy/openstack/openstack_deprovision.go index e973b2f43..81e1e12b6 100644 --- a/pkg/destroy/openstack/openstack_deprovision.go +++ b/pkg/destroy/openstack/openstack_deprovision.go @@ -329,51 +329,61 @@ func deleteRouters(opts *clientconfig.ClientOpts, filter Filter, logger logrus.F os.Exit(1) } for _, router := range allRouters { - // Get non HA router interface ports - portListOpts := ports.ListOpts{ - DeviceID: router.ID, - DeviceOwner: "network:router_interface", + // If a user provisioned floating ip was used, it needs to be dissociated + // Any floating Ip's associated with routers that are going to be deleted will be dissociated + fipOpts := floatingips.ListOpts{ + RouterID: router.ID, } - allPagesPort, err := ports.List(conn, portListOpts).AllPages() + + fipPages, err := floatingips.List(conn, fipOpts).AllPages() if err != nil { logger.Fatalf("%v", err) os.Exit(1) } - allPorts, err := ports.ExtractPorts(allPagesPort) + + allFIPs, err := floatingips.ExtractFloatingIPs(fipPages) if err != nil { logger.Fatalf("%v", err) os.Exit(1) } - // Get HA router interface ports - HAportListOpts := ports.ListOpts{ - DeviceID: router.ID, - DeviceOwner: "network:ha_router_replicated_interface", + for _, fip := range allFIPs { + _, err := floatingips.Update(conn, fip.ID, floatingips.UpdateOpts{}).Extract() + if err != nil { + logger.Fatalf("%v", err) + } + } + + // Get router interface ports + portListOpts := ports.ListOpts{ + DeviceID: router.ID, } - HAallPagesPort, err := ports.List(conn, HAportListOpts).AllPages() + allPagesPort, err := ports.List(conn, portListOpts).AllPages() if err != nil { logger.Fatalf("%v", err) os.Exit(1) } - HAPorts, err := ports.ExtractPorts(HAallPagesPort) + allPorts, err := ports.ExtractPorts(allPagesPort) if err != nil { logger.Fatalf("%v", err) os.Exit(1) } - // Catch all, since router may not be HA - allPorts = append(allPorts, HAPorts...) - + // map to keep track of whethere interface for subnet was already removed + removedSubnets := make(map[string]bool) for _, port := range allPorts { for _, IP := range port.FixedIPs { - removeOpts := routers.RemoveInterfaceOpts{ - SubnetID: IP.SubnetID, - } - logger.Debugf("Removing Subnet %v from Router %v\n", IP.SubnetID, router.ID) - _, err = routers.RemoveInterface(conn, router.ID, removeOpts).Extract() - if err != nil { - // This can fail when subnet is still in use - return false, nil + if !removedSubnets[IP.SubnetID] { + removeOpts := routers.RemoveInterfaceOpts{ + SubnetID: IP.SubnetID, + } + logger.Debugf("Removing Subnet %v from Router %v\n", IP.SubnetID, router.ID) + _, err = routers.RemoveInterface(conn, router.ID, removeOpts).Extract() + if err != nil { + // This can fail when subnet is still in use + return false, nil + } + removedSubnets[IP.SubnetID] = true } } } From 909eca838448cf60e8b9760f3bb7ebdb2056c209 Mon Sep 17 00:00:00 2001 From: Michal Fojtik Date: Tue, 14 May 2019 17:06:01 +0200 Subject: [PATCH 6/8] tls: add openshift service to DNS names --- pkg/asset/tls/apiserver.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/asset/tls/apiserver.go b/pkg/asset/tls/apiserver.go index 908b738b1..a50bee836 100644 --- a/pkg/asset/tls/apiserver.go +++ b/pkg/asset/tls/apiserver.go @@ -293,6 +293,9 @@ func (a *KubeAPIServerServiceNetworkServerCertKey) Generate(dependencies asset.P "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster.local", + "openshift", "openshift.default", + "openshift.default.svc", + "openshift.default.svc.cluster.local", }, IPAddresses: []net.IP{net.ParseIP(serviceAddress)}, } From d08972d5daff6308f996876782b921f3a99a2af2 Mon Sep 17 00:00:00 2001 From: Abhinav Dahiya Date: Mon, 13 May 2019 13:18:24 -0700 Subject: [PATCH 7/8] data: add approve-csr service to approve CSRs until bootstrap is complete PR for cluster-machine-approver [1] is taking over the approval of CSRs for client certificates for Machines that end up as Nodes in Openshift clusters. But during bootstrapping, cluster-machine-approver is not available and therefore, this service is required to approve CSRs until we have successfully bootstrapped the control plane, after which cluster-machine-approver or users take over the role of approving any new CSRs. Currently, all CSRs are automatically approved without any condition, this PR scopes it to only during bootstrapping phase, securing the endpoint for later use. [1]: https://github.com/openshift/cluster-machine-approver/pull/26 --- .../bootstrap/files/usr/local/bin/approve-csr.sh | 12 ++++++++++++ .../bootstrap/systemd/units/approve-csr.service | 13 +++++++++++++ pkg/asset/ignition/bootstrap/bootstrap.go | 1 + 3 files changed, 26 insertions(+) create mode 100644 data/data/bootstrap/files/usr/local/bin/approve-csr.sh create mode 100644 data/data/bootstrap/systemd/units/approve-csr.service diff --git a/data/data/bootstrap/files/usr/local/bin/approve-csr.sh b/data/data/bootstrap/files/usr/local/bin/approve-csr.sh new file mode 100644 index 000000000..479556a75 --- /dev/null +++ b/data/data/bootstrap/files/usr/local/bin/approve-csr.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +KUBECONFIG="${1}" + +echo "Approving all CSR requests until bootstrapping is complete..." +while [ ! -f /opt/openshift/bootkube.done ] +do + oc --config="$KUBECONFIG" get csr --no-headers | grep Pending | \ + awk '{print $1}' | \ + xargs --no-run-if-empty oc --config="$KUBECONFIG" adm certificate approve + sleep 20 +done diff --git a/data/data/bootstrap/systemd/units/approve-csr.service b/data/data/bootstrap/systemd/units/approve-csr.service new file mode 100644 index 000000000..abd943b62 --- /dev/null +++ b/data/data/bootstrap/systemd/units/approve-csr.service @@ -0,0 +1,13 @@ +[Unit] +Description=Approve CSRs during bootstrap phase +Wants=bootkube.service +After=bootkube.service + +[Service] +ExecStart=/usr/local/bin/approve-csr.sh /opt/openshift/auth/kubeconfig + +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=multi-user.target diff --git a/pkg/asset/ignition/bootstrap/bootstrap.go b/pkg/asset/ignition/bootstrap/bootstrap.go index ef55eb7d5..9c14d5555 100644 --- a/pkg/asset/ignition/bootstrap/bootstrap.go +++ b/pkg/asset/ignition/bootstrap/bootstrap.go @@ -248,6 +248,7 @@ func (a *Bootstrap) addSystemdUnits(uri string, templateData *bootstrapTemplateD "kubelet.service": {}, "chown-gatewayd-key.service": {}, "systemd-journal-gatewayd.socket": {}, + "approve-csr.service": {}, } directory, err := data.Assets.Open(uri) From 88f9d6bd9198ca327659055327af189552be4898 Mon Sep 17 00:00:00 2001 From: Abhinav Dahiya Date: Mon, 13 May 2019 15:28:43 -0700 Subject: [PATCH 8/8] data: capture approve-csr logs during bootstrap-gather --- data/data/bootstrap/files/usr/local/bin/installer-gather.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/data/bootstrap/files/usr/local/bin/installer-gather.sh b/data/data/bootstrap/files/usr/local/bin/installer-gather.sh index eb3d1dc33..7e4e1f584 100755 --- a/data/data/bootstrap/files/usr/local/bin/installer-gather.sh +++ b/data/data/bootstrap/files/usr/local/bin/installer-gather.sh @@ -4,7 +4,7 @@ ARTIFACTS="/tmp/artifacts" echo "Gathering bootstrap journals ..." mkdir -p "${ARTIFACTS}/bootstrap/journals" -for service in bootkube openshift kubelet crio +for service in bootkube openshift kubelet crio approve-csr do journalctl --boot --no-pager --output=short --unit="${service}" > "${ARTIFACTS}/bootstrap/journals/${service}.log" done