Skip to content

Commit

Permalink
ci: add Rancher Manager upgrade test
Browse files Browse the repository at this point in the history
Signed-off-by: Loic Devulder <ldevulder@suse.com>
  • Loading branch information
ldevulder committed Jun 14, 2023
1 parent 1ded7b5 commit 19352ea
Show file tree
Hide file tree
Showing 12 changed files with 224 additions and 105 deletions.
3 changes: 1 addition & 2 deletions .github/workflows/cli-k3s-os-upgrade-rancher_latest.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,7 @@ jobs:
cluster_name: cluster-k3s
iso_to_test: https://download.opensuse.org/repositories/isv:/Rancher:/Elemental:/Stable:/Teal53/media/iso/elemental-teal.x86_64.iso
k8s_version_to_provision: v1.25.7+k3s1
# Force to only deploy the first 3 nodes
node_number: 3
node_number: 5
operator_upgrade: oci://registry.opensuse.org/isv/rancher/elemental/dev/charts/rancher
operator_repo: oci://registry.opensuse.org/isv/rancher/elemental/stable/charts/rancher
rancher_version: latest/devel
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/cli-k3s-os-upgrade-rancher_stable.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@ jobs:
cluster_name: cluster-k3s
iso_to_test: https://download.opensuse.org/repositories/isv:/Rancher:/Elemental:/Stable:/Teal53/media/iso/elemental-teal.x86_64.iso
k8s_version_to_provision: v1.25.7+k3s1
# Force to only deploy the first 3 nodes
node_number: 3
node_number: 5
operator_upgrade: oci://registry.opensuse.org/isv/rancher/elemental/dev/charts/rancher
operator_repo: oci://registry.opensuse.org/isv/rancher/elemental/stable/charts/rancher
rancher_upgrade: latest/devel
rancher_version: stable/latest
upgrade_channel_list: registry.opensuse.org/isv/rancher/elemental/dev/teal53/15.4/rancher/elemental-teal-dev-channel/5.3:latest
upgrade_image: registry.opensuse.org/isv/rancher/elemental/dev/teal53/15.4/rancher/elemental-teal/5.3:latest
Expand Down
8 changes: 6 additions & 2 deletions .github/workflows/cli-k3s-os-upgrade.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ on:
description: ISO to test
default: https://download.opensuse.org/repositories/isv:/Rancher:/Elemental:/Stable:/Teal53/media/iso/elemental-teal.x86_64.iso
type: string
rancher_upgrade:
description: Rancher Manager channel/version to upgrade to
default: latest/devel
type: string
rancher_version:
description: Rancher Manager channel/version to use for installation
default: stable/latest
Expand All @@ -38,10 +42,10 @@ jobs:
destroy_runner: ${{ inputs.destroy_runner }}
iso_to_test: ${{ inputs.iso_to_test }}
k8s_version_to_provision: v1.25.7+k3s1
# Force to only deploy the first 3 nodes
node_number: 3
node_number: 5
operator_upgrade: oci://registry.opensuse.org/isv/rancher/elemental/${{ inputs.upgrade_os_channel }}/charts/rancher
operator_repo: oci://registry.opensuse.org/isv/rancher/elemental/stable/charts/rancher
rancher_upgrade: ${{ inputs.rancher_upgrade }}
rancher_version: ${{ inputs.rancher_version }}
upgrade_channel_list: registry.opensuse.org/isv/rancher/elemental/dev/teal53/15.4/rancher/elemental-teal-dev-channel/5.3:latest
upgrade_image: registry.opensuse.org/isv/rancher/elemental/${{ inputs.upgrade_os_channel }}/teal53/15.4/rancher/elemental-teal/5.3:latest
Expand Down
3 changes: 1 addition & 2 deletions .github/workflows/cli-rke2-os-upgrade-rancher_latest.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,7 @@ jobs:
cluster_name: cluster-rke2
iso_to_test: https://download.opensuse.org/repositories/isv:/Rancher:/Elemental:/Stable:/Teal53/media/iso/elemental-teal.x86_64.iso
k8s_version_to_provision: v1.25.7+rke2r1
# Force to only deploy the first 3 nodes
node_number: 3
node_number: 5
operator_upgrade: oci://registry.opensuse.org/isv/rancher/elemental/dev/charts/rancher
operator_repo: oci://registry.opensuse.org/isv/rancher/elemental/stable/charts/rancher
rancher_version: latest/devel
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/cli-rke2-os-upgrade-rancher_stable.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@ jobs:
cluster_name: cluster-rke2
iso_to_test: https://download.opensuse.org/repositories/isv:/Rancher:/Elemental:/Stable:/Teal53/media/iso/elemental-teal.x86_64.iso
k8s_version_to_provision: v1.25.7+rke2r1
# Force to only deploy the first 3 nodes
node_number: 3
node_number: 5
operator_upgrade: oci://registry.opensuse.org/isv/rancher/elemental/dev/charts/rancher
operator_repo: oci://registry.opensuse.org/isv/rancher/elemental/stable/charts/rancher
rancher_upgrade: latest/devel
rancher_version: stable/latest
upgrade_channel_list: registry.opensuse.org/isv/rancher/elemental/dev/teal53/15.4/rancher/elemental-teal-dev-channel/5.3:latest
upgrade_image: registry.opensuse.org/isv/rancher/elemental/dev/teal53/15.4/rancher/elemental-teal/5.3:latest
Expand Down
8 changes: 6 additions & 2 deletions .github/workflows/cli-rke2-os-upgrade.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ on:
description: ISO to test
default: https://download.opensuse.org/repositories/isv:/Rancher:/Elemental:/Stable:/Teal53/media/iso/elemental-teal.x86_64.iso
type: string
rancher_upgrade:
description: Rancher Manager channel/version to upgrade to
default: latest/devel
type: string
rancher_version:
description: Rancher Manager channel/version to use for installation
default: stable/latest
Expand Down Expand Up @@ -39,10 +43,10 @@ jobs:
destroy_runner: ${{ inputs.destroy_runner }}
iso_to_test: ${{ inputs.iso_to_test }}
k8s_version_to_provision: v1.25.7+rke2r1
# Force to only deploy the first 3 nodes
node_number: 3
node_number: 5
operator_upgrade: oci://registry.opensuse.org/isv/rancher/elemental/${{ inputs.upgrade_os_channel }}/charts/rancher
operator_repo: oci://registry.opensuse.org/isv/rancher/elemental/stable/charts/rancher
rancher_upgrade: ${{ inputs.rancher_upgrade }}
rancher_version: ${{ inputs.rancher_version }}
upgrade_channel_list: registry.opensuse.org/isv/rancher/elemental/dev/teal53/15.4/rancher/elemental-teal-dev-channel/5.3:latest
upgrade_image: registry.opensuse.org/isv/rancher/elemental/${{ inputs.upgrade_os_channel }}/teal53/15.4/rancher/elemental-teal/5.3:latest
Expand Down
25 changes: 24 additions & 1 deletion .github/workflows/master-e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,9 @@ on:
description: Rancher Manager channel/version to use for installation
default: stable/latest
type: string
rancher_upgrade:
description: Rancher Manager channel/version to upgrade to
type: string
runner_template:
description: Runner template to use
default: elemental-e2e-ci-runner-spot-x86-64-template-n2-standard-16-v4
Expand Down Expand Up @@ -265,9 +268,10 @@ jobs:
CA_TYPE: ${{ inputs.ca_type }}
OPERATOR_REPO: ${{ inputs.operator_repo }}
PROXY: ${{ inputs.proxy }}
PUBLIC_DNS: ${{ needs.create-runner.outputs.public_dns }}
PUBLIC_DOMAIN: bc.googleusercontent.com
TEST_TYPE: ${{ inputs.test_type }}
run: cd tests && PUBLIC_DNS=${{ needs.create-runner.outputs.public_dns }} make e2e-install-rancher
run: cd tests && make e2e-install-rancher
- name: Workaround for DynamicSchemas (if needed)
run: |
# Check if DynamicSchemas for MachineInventorySelectorTemplate exists
Expand Down Expand Up @@ -459,6 +463,24 @@ jobs:
# Export values
echo "operator_upgrade=${OPERATOR_UPGRADE}" >> ${GITHUB_OUTPUT}
echo "operator_version=${OPERATOR_VERSION}" >> ${GITHUB_OUTPUT}
- name: Upgrade Rancher Manager
if: inputs.test_type == 'cli' && inputs.rancher_upgrade != ''
id: rancher_upgrade
env:
CA_TYPE: ${{ inputs.ca_type }}
PROXY: ${{ inputs.proxy }}
PUBLIC_DNS: ${{ needs.create-runner.outputs.public_dns }}
PUBLIC_DOMAIN: bc.googleusercontent.com
RANCHER_UPGRADE: ${{ inputs.rancher_upgrade }}
run: |
cd tests && make e2e-upgrade-rancher-manager
# Extract Rancher Manager version
RM_VERSION=$(kubectl get pod \
--namespace cattle-system \
-l app=rancher \
-o jsonpath={.items[*].status.containerStatuses[*].image} 2> /dev/null || true)
# Export values
echo "rm_version=${RM_VERSION}" >> ${GITHUB_OUTPUT}
- name: Upgrade node 1 to specified OS version with osImage
if: inputs.test_type == 'cli' && inputs.upgrade_image != ''
env:
Expand Down Expand Up @@ -577,6 +599,7 @@ jobs:
echo "## Upgrade details" >> ${GITHUB_STEP_SUMMARY}
echo "Elemental Operator Upgrade: ${{ steps.operator_upgrade.outputs.operator_upgrade }}" >> ${GITHUB_STEP_SUMMARY}
echo "Elemental Operator Image: ${{ steps.operator_upgrade.outputs.operator_version }}" >> ${GITHUB_STEP_SUMMARY}
echo "Rancher Manager Version: ${{ steps.rancher_upgrade.outputs.rm_version }}" >> ${GITHUB_STEP_SUMMARY}
echo "Channel list: ${{ inputs.upgrade_channel_list }}" >> ${GITHUB_STEP_SUMMARY}
echo "Channel used: ${{ inputs.upgrade_os_channel }}" >> ${GITHUB_STEP_SUMMARY}
echo "Upgrade image: ${{ inputs.upgrade_image }}" >> ${GITHUB_STEP_SUMMARY}
Expand Down
4 changes: 3 additions & 1 deletion tests/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,10 @@ e2e-ui-rancher: deps
e2e-uninstall-operator:
ginkgo --label-filter uninstall-operator -r -v ./e2e
e2e-upgrade-node: deps
ginkgo --label-filter upgrade -r -v ./e2e
ginkgo --label-filter upgrade-node -r -v ./e2e
e2e-upgrade-operator: deps
ginkgo --label-filter upgrade-operator -r -v ./e2e
e2e-upgrade-rancher-manager: deps
ginkgo --label-filter upgrade-rancher-manager -r -v ./e2e
start-cypress-tests:
@./scripts/start-cypress-tests
86 changes: 86 additions & 0 deletions tests/e2e/helpers/install/install.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
/*
Copyright © 2023 SUSE LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package install

import (
"strings"

. "github.com/onsi/gomega"
"github.com/rancher-sandbox/ele-testhelpers/kubectl"
)

// Install or upgrade Rancher Manager
func DeployRancherManager(hostname, channel, version, ca, proxy string) {
channelName := "rancher-" + channel

// Add Helm repository
err := kubectl.RunHelmBinaryWithCustomErr("repo", "add", channelName,
"https://releases.rancher.com/server-charts/"+channel,
)
Expect(err).To(Not(HaveOccurred()))

err = kubectl.RunHelmBinaryWithCustomErr("repo", "update")
Expect(err).To(Not(HaveOccurred()))

// Set flags for Rancher Manager installation
flags := []string{
"upgrade", "--install", "rancher", channelName + "/rancher",
"--namespace", "cattle-system",
"--create-namespace",
"--set", "hostname=" + hostname,
"--set", "extraEnv[0].name=CATTLE_SERVER_URL",
"--set", "extraEnv[0].value=https://" + hostname,
"--set", "extraEnv[1].name=CATTLE_BOOTSTRAP_PASSWORD",
"--set", "extraEnv[1].value=rancherpassword",
"--set", "replicas=1",
"--set", "global.cattle.psp.enabled=false",
}

// Set specified version if needed
if version != "" && version != "latest" {
if version == "devel" {
flags = append(flags,
"--devel",
"--set", "rancherImageTag=v2.7-head",
)
} else if strings.Contains(version, "-rc") {
flags = append(flags,
"--devel",
"--version", version,
)
} else {
flags = append(flags, "--version", version)
}
}

// For Private CA
if ca == "private" {
flags = append(flags,
"--set", "ingress.tls.source=secret",
"--set", "privateCA=true",
)
}

// Use Rancher Manager behind proxy
if proxy == "rancher" {
flags = append(flags,
"--set", "proxy=http://172.17.0.1:3128",
"--set", "noProxy=127.0.0.0/8\\,10.0.0.0/8\\,cattle-system.svc\\,172.16.0.0/12\\,192.168.0.0/16\\,.svc\\,.cluster.local",
)
}

err = kubectl.RunHelmBinaryWithCustomErr(flags...)
Expect(err).To(Not(HaveOccurred()))
}
59 changes: 2 additions & 57 deletions tests/e2e/install_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
. "github.com/onsi/gomega"
"github.com/rancher-sandbox/ele-testhelpers/kubectl"
"github.com/rancher-sandbox/ele-testhelpers/tools"
"github.com/rancher/elemental/tests/e2e/helpers/install"
"github.com/rancher/elemental/tests/e2e/helpers/misc"
)

Expand Down Expand Up @@ -190,63 +191,7 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() {
}

By("Installing Rancher", func() {
err := kubectl.RunHelmBinaryWithCustomErr("repo", "add", "rancher",
"https://releases.rancher.com/server-charts/"+rancherChannel,
)
Expect(err).To(Not(HaveOccurred()))

err = kubectl.RunHelmBinaryWithCustomErr("repo", "update")
Expect(err).To(Not(HaveOccurred()))

// Set flags for Rancher Manager installation
flags := []string{
"upgrade", "--install", "rancher", "rancher/rancher",
"--namespace", "cattle-system",
"--create-namespace",
"--set", "hostname=" + rancherHostname,
"--set", "extraEnv[0].name=CATTLE_SERVER_URL",
"--set", "extraEnv[0].value=https://" + rancherHostname,
"--set", "extraEnv[1].name=CATTLE_BOOTSTRAP_PASSWORD",
"--set", "extraEnv[1].value=rancherpassword",
"--set", "replicas=1",
"--set", "global.cattle.psp.enabled=false",
}

// Set specified version if needed
if rancherVersion != "" && rancherVersion != "latest" {
if rancherVersion == "devel" {
flags = append(flags,
"--devel",
"--set", "rancherImageTag=v2.7-head",
)
} else if strings.Contains(rancherVersion, "-rc") {
flags = append(flags,
"--devel",
"--version", rancherVersion,
)
} else {
flags = append(flags, "--version", rancherVersion)
}
}

// For Private CA
if caType == "private" {
flags = append(flags,
"--set", "ingress.tls.source=secret",
"--set", "privateCA=true",
)
}

// Use Rancher Manager behind proxy
if proxy == "rancher" {
flags = append(flags,
"--set", "proxy=http://172.17.0.1:3128",
"--set", "noProxy=127.0.0.0/8\\,10.0.0.0/8\\,cattle-system.svc\\,172.16.0.0/12\\,192.168.0.0/16\\,.svc\\,.cluster.local",
)
}

err = kubectl.RunHelmBinaryWithCustomErr(flags...)
Expect(err).To(Not(HaveOccurred()))
install.DeployRancherManager(rancherHostname, rancherChannel, rancherVersion, caType, proxy)

// Inject secret for Private CA
if caType == "private" {
Expand Down
Loading

0 comments on commit 19352ea

Please sign in to comment.