From 382e1724f6141adca77ddc33fc2262540750f524 Mon Sep 17 00:00:00 2001 From: Varad Ahirwadkar Date: Thu, 1 Sep 2022 04:41:58 -0700 Subject: [PATCH] Adding eus to eus ocp upgrade Signed-off-by: Varad Ahirwadkar --- examples/upgrade_vars.yaml | 4 + playbooks/roles/ocp-upgrade/README.md | 14 +- .../roles/ocp-upgrade/defaults/main.yaml | 4 + .../roles/ocp-upgrade/tasks/eus_upgrades.yaml | 196 ++++++++++++++++++ playbooks/roles/ocp-upgrade/tasks/main.yaml | 16 +- playbooks/upgrade.yaml | 4 +- 6 files changed, 232 insertions(+), 6 deletions(-) create mode 100644 playbooks/roles/ocp-upgrade/tasks/eus_upgrades.yaml diff --git a/examples/upgrade_vars.yaml b/examples/upgrade_vars.yaml index b1711dae..ad8ac60b 100644 --- a/examples/upgrade_vars.yaml +++ b/examples/upgrade_vars.yaml @@ -1,4 +1,8 @@ --- +eus_intermediate_upgrade_version: "" #OCP upgrade version eg. 4.11.2 +eus_upgrade_channel: "" #OCP channel having required upgrade version available for cluster upgrade (stable-4.x, fast-4.x, candidate-4.x, eus-4.x) eg. stable-4.11 +eus_intermediate_upgrade_image: "" #OCP upgrade image e.g. "quay.io/openshift-release-dev/ocp-release-nightly@sha256:xxxxx" +upstream: "" #Set the URL for OCP update server eg. https://ppc64le.ocp.releases.ci.openshift.org/graph upgrade_version: "" #OCP upgrade version eg. 4.5.4 upgrade_channel: "" #OCP channel having required upgrade version available for cluster upgrade (stable-4.x, fast-4.x, candidate-4.x) eg. stable-4.5 upgrade_image: "" #OCP upgrade image e.g. "quay.io/openshift-release-dev/ocp-release-nightly@sha256:xxxxx" diff --git a/playbooks/roles/ocp-upgrade/README.md b/playbooks/roles/ocp-upgrade/README.md index d0122a61..44dc664d 100644 --- a/playbooks/roles/ocp-upgrade/README.md +++ b/playbooks/roles/ocp-upgrade/README.md @@ -1,7 +1,9 @@ ocp-upgrade: Upgrade OCP cluster ========= -This module will upgrade an existing OCP cluster based on upgrade_version or upgrade_image. If both upgrade_version and upgrade_image are specified then upgrade_image value gets preference. +This module will upgrade an existing OCP cluster based on upgrade_version or upgrade_image. If both upgrade_version and upgrade_image are specified then upgrade_image value gets preference. + +For intermediate EUS upgrade please use eus variables and use other variables for final upgrade. Requirements ------------ @@ -13,12 +15,18 @@ Role Variables | Variable | Required | Default | Comments | |-----------------|----------|------------|---------------------------------------------------------------| +| eus_intermediate_upgrade_version | no | "" | Set to a specific version eg. 4.11.3 | +| eus_upgrade_channel | no | "" | Set to channel having required upgrade version available for cluster upgrade (stable-4.x, fast-4.x, candidate-4.x, eus-4.x) eg. stable-4.11 | +| eus_intermediate_upgrade_image | no | "" | Set to OCP upgrade image eg. quay.io/openshift-release-dev/ocp-release@sha256:12345.. | +| upstream | no | "" | Set the URL for OCP update server eg. https://ppc64le.ocp.releases.ci.openshift.org/graph | | upgrade_version | no | "" | Set to a specific version eg. 4.5.4 | | upgrade_channel | no | "" | Set to channel having required upgrade version available for cluster upgrade (stable-4.x, fast-4.x, candidate-4.x) eg. stable-4.5 | | upgrade_image | no | "" | Set to OCP upgrade image eg. quay.io/openshift-release-dev/ocp-release@sha256:12345.. | | pause_time | no | 90 | Pauses playbook execution for a set amount of time in minutes | | delay_time | no | 600 | Number of seconds to wait before starting to poll | +Note: If eus_upgrade_channel is set to the eus channel then no need to set upgrade_channel. + Dependencies ------------ @@ -32,7 +40,9 @@ Example Playbook tasks: - include_role: name: ocp-upgrade - when: upgrade_version != "" + when: > + (upgrade_version != "") or (upgrade_image != "") or + (eus_intermediate_upgrade_channel != "") or (eus_intermediate_upgrade_image != "") License ------- diff --git a/playbooks/roles/ocp-upgrade/defaults/main.yaml b/playbooks/roles/ocp-upgrade/defaults/main.yaml index b61b91a2..480ba808 100644 --- a/playbooks/roles/ocp-upgrade/defaults/main.yaml +++ b/playbooks/roles/ocp-upgrade/defaults/main.yaml @@ -1,6 +1,10 @@ --- # defaults file +eus_intermediate_upgrade_version: "" +eus_upgrade_channel: "" +eus_intermediate_upgrade_image: "" +upstream: "" upgrade_version: "" upgrade_channel: "" upgrade_image: "" diff --git a/playbooks/roles/ocp-upgrade/tasks/eus_upgrades.yaml b/playbooks/roles/ocp-upgrade/tasks/eus_upgrades.yaml new file mode 100644 index 00000000..46b4ffb6 --- /dev/null +++ b/playbooks/roles/ocp-upgrade/tasks/eus_upgrades.yaml @@ -0,0 +1,196 @@ +--- + +- name: Pause the worker's MachineConfigPool + kubernetes.core.k8s: + state: present + definition: + kind: MachineConfigPool + metadata: + name: worker + spec: + paused: true + +- name: Patch the ClusterVersion with the eus intermediate upgrade channel + kubernetes.core.k8s: + state: present + definition: + kind: ClusterVersion + metadata: + name: version + spec: + channel: "{{ eus_upgrade_channel }}" + upstream: "{{ upstream }}" + when: eus_upgrade_channel != "" + +- name: Get the OCP version + kubernetes.core.k8s_info: + kind: ClusterVersion + name: version + register: ocp_version + +- name: Providing the administrator acknowledgement for 4.8 to 4.9 upgrades + kubernetes.core.k8s: + state: present + definition: + kind: ConfigMap + metadata: + name: admin-acks + namespace: openshift-config + data: + ack-4.8-kube-1.22-api-removals-in-4.9: "true" + when: ocp_version.resources[0].status.history[0].version | regex_search('4.8') == "4.8" + +- name: Upgrade cluster with upgrade version + shell: oc adm upgrade --to {{ eus_intermediate_upgrade_version }} + when: eus_intermediate_upgrade_image == "" and eus_intermediate_upgrade_version != "" + +- name: Upgrade cluster with intermediate upgrade image + shell: | + oc adm upgrade --force --to-image="{{ eus_intermediate_upgrade_image }}" --allow-explicit-upgrade + when: eus_intermediate_upgrade_image != "" + +- name: Upgrade the cluster to the latest available build in the channel + shell: oc adm upgrade --to-latest + when: + - eus_upgrade_channel != "" and eus_intermediate_upgrade_image == "" + - eus_intermediate_upgrade_version == "" + +- name: Waiting until upgrade completes + pause: + minutes: "{{ pause_time }}" + +- name: Check for upgrade completion + shell: oc get clusterversion -o json|jq ".items[0].status.history[0].state" + register: state + until: state.stdout | replace('\"', '') == "Completed" + retries: 5 + delay: "{{ delay_time }}" + +- name: Check all co are in 'Available' state and not in 'Progressing' or 'Degraded' state + shell: oc get co --no-headers | awk '{ print $3 $4 $5 }' | grep -w -v TrueFalseFalse | wc -l + register: co_count + until: 0 == co_count.stdout|int + retries: 6 + delay: 300 + +- name: Check all node are healthy + shell: oc get nodes --no-headers | grep -v Ready | wc -l + register: nodes_count + until: 0 == nodes_count.stdout|int + retries: 6 + delay: 300 + +- name: Get the master's MachineConfigPool information + kubernetes.core.k8s_info: + kind: MachineConfigPool + name: master + register: master_mcp + +- name: Check if the master's MachineConfigPool has updated + fail: + msg: "Master MachineConfigPool not get updated" + when: master_mcp.resources[0].status.updatedMachineCount != master_mcp.resources[0].status.machineCount + +- name: Get the worker's MachineConfigPool information + kubernetes.core.k8s_info: + kind: MachineConfigPool + name: worker + register: worker_mcp + +- name: Check if the worker's MachineConfigPool has not updated + fail: + msg: "Worker machine config pool get updated" + when: worker_mcp.resources[0].status.updatedMachineCount != 0 + +- name: Patch the ClusterVersion with the upgrade channel + kubernetes.core.k8s: + state: present + definition: + kind: ClusterVersion + metadata: + name: version + spec: + channel: "{{ upgrade_channel }}" + upstream: "{{ upstream }}" + when: upgrade_channel != "" + +- name: Upgrade cluster with final upgrade version + shell: oc adm upgrade --to {{ upgrade_version }} + when: upgrade_image == "" and upgrade_version != "" + +- name: Upgrade cluster with final upgrade image + shell: oc adm upgrade --force --to-image="{{ upgrade_image }}" --allow-explicit-upgrade + when: upgrade_image != "" + +- name: Upgrade the cluster to the latest available build in the channel + shell: oc adm upgrade --to-latest + when: > + (upgrade_channel != "" and upgrade_image == "" and upgrade_version == "") or + "eus" in eus_upgrade_channel + +- name: Waiting until upgrade completes + pause: + minutes: "{{ pause_time }}" + +- name: Check for upgrade completion + shell: oc get clusterversion -o json|jq ".items[0].status.history[0].state" + register: state + until: state.stdout | replace('\"', '') == "Completed" + retries: 5 + delay: "{{ delay_time }}" + +- name: Check all co are in 'Available' state and not in 'Progressing' or 'Degraded' state + shell: oc get co --no-headers | awk '{ print $3 $4 $5 }' | grep -w -v TrueFalseFalse | wc -l + register: co_count + until: 0 == co_count.stdout|int + retries: 6 + delay: 300 + +- name: Check all node are healthy + shell: oc get nodes --no-headers | grep -v Ready | wc -l + register: nodes_count + until: 0 == nodes_count.stdout|int + retries: 6 + delay: 300 + +- name: Get the updated machine count of masters + kubernetes.core.k8s_info: + kind: MachineConfigPool + name: master + register: master_mcp + +- name: Check if the master's MachineConfigPool has updated + fail: + msg: "Master MachineConfigPool not get updated" + when: master_mcp.resources[0].status.updatedMachineCount != master_mcp.resources[0].status.machineCount + +- name: Resume the worker's MachineConfigPool + kubernetes.core.k8s: + state: present + definition: + kind: MachineConfigPool + metadata: + name: worker + spec: + paused: false + +- name: Check if the worker's MachineConfigPool is updating + shell: oc get mcp worker -o jsonpath='{.status.updatedMachineCount}' + register: worker_updated_machine_count + until: worker_updated_machine_count.stdout|int > 0 + retries: 3 + delay: 600 + +- name: Check if the worker's MachineConfigPool has updated + shell: oc get mcp worker -o jsonpath='{.status.updatedMachineCount}' + register: worker_updated_machine_count + until: worker_updated_machine_count.stdout|int == worker_mcp.resources[0].status.machineCount|int + retries: 5 + delay: 600 + +- name: Check all co are in 'Available' state and not in 'Progressing' or 'Degraded' state + shell: oc get co --no-headers | awk '{ print $3 $4 $5 }' | grep -w -v TrueFalseFalse | wc -l + register: co_count + until: 0 == co_count.stdout|int + retries: 6 + delay: 300 diff --git a/playbooks/roles/ocp-upgrade/tasks/main.yaml b/playbooks/roles/ocp-upgrade/tasks/main.yaml index 1dab0535..7a649f41 100644 --- a/playbooks/roles/ocp-upgrade/tasks/main.yaml +++ b/playbooks/roles/ocp-upgrade/tasks/main.yaml @@ -1,7 +1,7 @@ --- - name: Check all co are in 'Available' state and not in 'Progressing' or 'Degraded' state - shell: oc get co --no-headers | awk '{ print $3 $4 $5 }' | grep -w -v TrueFalseFalse | wc -l + shell: oc get co --no-headers | awk '{ print $3 $4 $5 }' | grep -w -v TrueFalseFalse | wc -l register: cocount until: 0 == cocount.stdout|int retries: 6 @@ -34,7 +34,9 @@ until: state.stdout | replace('\"', '') == "Completed" retries: 5 delay: "{{ delay_time }}" - when: upgrade_version != "" and upgrade_image == "" + when: + - upgrade_version != "" and upgrade_image == "" + - eus_upgrade_channel == "" and eus_intermediate_upgrade_image == "" - name: Upgrade ocp using a release image block: @@ -53,4 +55,12 @@ until: state.stdout | replace('\"', '') == "Completed" retries: 5 delay: "{{ delay_time }}" - when: upgrade_image != "" + when: + - upgrade_image != "" + - eus_upgrade_channel == "" and eus_intermediate_upgrade_image == "" + +- name: EUS to EUS upgrades + include_tasks: "{{ role_path }}/tasks/eus_upgrades.yaml" + when: > + (eus_upgrade_channel != "" or eus_intermediate_upgrade_image != "") and + ("eus" in eus_upgrade_channel or (upgrade_version != "" or upgrade_image != "" or upgrade_channel != "")) diff --git a/playbooks/upgrade.yaml b/playbooks/upgrade.yaml index 910f8578..3c029c7b 100644 --- a/playbooks/upgrade.yaml +++ b/playbooks/upgrade.yaml @@ -4,5 +4,7 @@ tasks: - include_role: name: ocp-upgrade - when: (upgrade_version != "") or (upgrade_image != "") + when: > + (upgrade_version != "") or (upgrade_image != "") or + (eus_upgrade_channel != "") or (eus_intermediate_upgrade_image != "")