-
Notifications
You must be signed in to change notification settings - Fork 60
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Loading status checks…
Add upgrade E2E
Testing upgrade from the latest release to the current commit Signed-off-by: Mikalai Radchuk <mradchuk@redhat.com>
- Loading branch information
Mikalai Radchuk
committed
Jul 10, 2024
1 parent
db5a75c
commit fe28a5b
Showing
5 changed files
with
276 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
#!/bin/bash | ||
|
||
set -euo pipefail | ||
|
||
help="pre-upgrade-setup.sh is used to create some basic resources | ||
which will later be used in upgrade testing. | ||
Usage: | ||
post-upgrade-checks.sh [TEST_CATALOG_IMG] [TEST_CATALOG_NAME] [TEST_CLUSTER_EXTENSION_NAME] | ||
" | ||
|
||
if [[ "$#" -ne 3 ]]; then | ||
echo "Illegal number of arguments passed" | ||
echo "${help}" | ||
exit 1 | ||
fi | ||
|
||
TEST_CATALOG_IMG=$1 | ||
TEST_CLUSTER_CATALOG_NAME=$2 | ||
TEST_CLUSTER_EXTENSION_NAME=$3 | ||
|
||
kubectl apply -f - << EOF | ||
apiVersion: catalogd.operatorframework.io/v1alpha1 | ||
kind: ClusterCatalog | ||
metadata: | ||
name: ${TEST_CLUSTER_CATALOG_NAME} | ||
spec: | ||
source: | ||
type: image | ||
image: | ||
ref: ${TEST_CATALOG_IMG} | ||
pollInterval: 24h | ||
insecureSkipTLSVerify: true | ||
EOF | ||
|
||
|
||
kubectl apply -f - << EOF | ||
apiVersion: olm.operatorframework.io/v1alpha1 | ||
kind: ClusterExtension | ||
metadata: | ||
name: ${TEST_CLUSTER_EXTENSION_NAME} | ||
spec: | ||
installNamespace: default | ||
packageName: prometheus | ||
version: 1.0.0 | ||
serviceAccount: | ||
name: default | ||
EOF | ||
|
||
kubectl wait --for=condition=Unpacked --timeout=60s ClusterCatalog $TEST_CLUSTER_CATALOG_NAME | ||
kubectl wait --for=condition=Installed --timeout=60s ClusterExtension $TEST_CLUSTER_EXTENSION_NAME |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,137 @@ | ||
package upgradee2e | ||
|
||
import ( | ||
"bufio" | ||
"context" | ||
"fmt" | ||
"strings" | ||
"testing" | ||
"time" | ||
|
||
"github.com/stretchr/testify/assert" | ||
"github.com/stretchr/testify/require" | ||
appsv1 "k8s.io/api/apps/v1" | ||
corev1 "k8s.io/api/core/v1" | ||
apimeta "k8s.io/apimachinery/pkg/api/meta" | ||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
"k8s.io/apimachinery/pkg/labels" | ||
"k8s.io/apimachinery/pkg/types" | ||
"sigs.k8s.io/controller-runtime/pkg/client" | ||
|
||
catalogdv1alpha1 "github.com/operator-framework/catalogd/api/core/v1alpha1" | ||
|
||
ocv1alpha1 "github.com/operator-framework/operator-controller/api/v1alpha1" | ||
) | ||
|
||
func TestClusterExtensionAfterOLMUpgrade(t *testing.T) { | ||
t.Log("Starting checks after OLM upgrade") | ||
ctx := context.Background() | ||
|
||
managerLabelSelector := labels.Set{"control-plane": "controller-manager"} | ||
|
||
t.Log("Checking that the controller-manager deployment is updated") | ||
require.EventuallyWithT(t, func(ct *assert.CollectT) { | ||
var managerDeployments appsv1.DeploymentList | ||
assert.NoError(ct, c.List(ctx, &managerDeployments, client.MatchingLabelsSelector{Selector: managerLabelSelector.AsSelector()})) | ||
assert.Len(ct, managerDeployments.Items, 1) | ||
managerDeployment := managerDeployments.Items[0] | ||
|
||
assert.True(ct, | ||
managerDeployment.Status.UpdatedReplicas == *managerDeployment.Spec.Replicas && | ||
managerDeployment.Status.Replicas == *managerDeployment.Spec.Replicas && | ||
managerDeployment.Status.AvailableReplicas == *managerDeployment.Spec.Replicas && | ||
managerDeployment.Status.ReadyReplicas == *managerDeployment.Spec.Replicas, | ||
) | ||
}, time.Minute, time.Second) | ||
|
||
var managerPod *corev1.Pod | ||
t.Log("Waiting for only one controller-manager pod to remain") | ||
require.EventuallyWithT(t, func(ct *assert.CollectT) { | ||
var managerPods corev1.PodList | ||
assert.NoError(ct, c.List(ctx, &managerPods, client.MatchingLabelsSelector{Selector: managerLabelSelector.AsSelector()})) | ||
assert.Len(ct, managerPods.Items, 1) | ||
|
||
managerPod = &managerPods.Items[0] | ||
}, time.Minute, time.Second) | ||
|
||
t.Log("Reading logs to make sure that ClusterExtension was reconciled by operator-controller before we update it") | ||
logCtx, cancel := context.WithTimeout(ctx, time.Minute) | ||
defer cancel() | ||
substring := fmt.Sprintf(`"ClusterExtension": {"name":"%s"}`, testClusterExtensionName) | ||
found, err := watchPodLogsForSubstring(logCtx, managerPod, "manager", substring) | ||
require.NoError(t, err) | ||
require.True(t, found) | ||
|
||
var clusterCatalog catalogdv1alpha1.ClusterCatalog | ||
var clusterExtension ocv1alpha1.ClusterExtension | ||
|
||
t.Log("Checking that the cluster catalog is unpacked") | ||
require.EventuallyWithT(t, func(ct *assert.CollectT) { | ||
assert.NoError(ct, c.Get(ctx, types.NamespacedName{Name: testClusterCatalogName}, &clusterCatalog)) | ||
cond := apimeta.FindStatusCondition(clusterCatalog.Status.Conditions, catalogdv1alpha1.TypeUnpacked) | ||
if !assert.NotNil(ct, cond) { | ||
return | ||
} | ||
assert.Equal(ct, metav1.ConditionTrue, cond.Status) | ||
assert.Equal(ct, catalogdv1alpha1.ReasonUnpackSuccessful, cond.Reason) | ||
}, time.Minute, time.Second) | ||
|
||
t.Log("Checking that the cluster extension is installed") | ||
require.EventuallyWithT(t, func(ct *assert.CollectT) { | ||
assert.NoError(ct, c.Get(ctx, types.NamespacedName{Name: testClusterExtensionName}, &clusterExtension)) | ||
cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1alpha1.TypeInstalled) | ||
if !assert.NotNil(ct, cond) { | ||
return | ||
} | ||
assert.Equal(ct, metav1.ConditionTrue, cond.Status) | ||
assert.Equal(ct, ocv1alpha1.ReasonSuccess, cond.Reason) | ||
assert.Contains(ct, cond.Message, "Instantiated bundle") | ||
assert.NotEmpty(ct, clusterExtension.Status.InstalledBundle) | ||
}, time.Minute, time.Second) | ||
|
||
previousVersion := clusterExtension.Status.InstalledBundle.Version | ||
|
||
t.Log("Updating the cluster extension to change version") | ||
// This is to make sure that after we upgrade OLM itself we can still reconcile old objects | ||
clusterExtension.Spec.Version = "1.0.1" | ||
require.NoError(t, c.Update(ctx, &clusterExtension)) | ||
|
||
t.Log("Checking that the cluster extension installs successfully") | ||
require.EventuallyWithT(t, func(ct *assert.CollectT) { | ||
assert.NoError(ct, c.Get(ctx, types.NamespacedName{Name: testClusterExtensionName}, &clusterExtension)) | ||
cond := apimeta.FindStatusCondition(clusterExtension.Status.Conditions, ocv1alpha1.TypeInstalled) | ||
if !assert.NotNil(ct, cond) { | ||
return | ||
} | ||
assert.Equal(ct, ocv1alpha1.ReasonSuccess, cond.Reason) | ||
assert.Contains(ct, cond.Message, "Instantiated bundle") | ||
assert.Equal(ct, &ocv1alpha1.BundleMetadata{Name: "prometheus-operator.1.0.1", Version: "1.0.1"}, clusterExtension.Status.ResolvedBundle) | ||
assert.Equal(ct, &ocv1alpha1.BundleMetadata{Name: "prometheus-operator.1.0.1", Version: "1.0.1"}, clusterExtension.Status.InstalledBundle) | ||
assert.NotEqual(ct, previousVersion, clusterExtension.Status.InstalledBundle.Version) | ||
}, time.Minute, time.Second) | ||
} | ||
|
||
func watchPodLogsForSubstring(ctx context.Context, pod *corev1.Pod, container, substring string) (bool, error) { | ||
podLogOpts := corev1.PodLogOptions{ | ||
Follow: true, | ||
Container: container, | ||
} | ||
|
||
req := kclientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts) | ||
podLogs, err := req.Stream(ctx) | ||
if err != nil { | ||
return false, err | ||
} | ||
defer podLogs.Close() | ||
|
||
scanner := bufio.NewScanner(podLogs) | ||
for scanner.Scan() { | ||
line := scanner.Text() | ||
|
||
if strings.Contains(line, substring) { | ||
return true, nil | ||
} | ||
} | ||
|
||
return false, scanner.Err() | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,57 @@ | ||
package upgradee2e | ||
|
||
import ( | ||
"fmt" | ||
"os" | ||
"testing" | ||
|
||
"k8s.io/client-go/kubernetes" | ||
ctrl "sigs.k8s.io/controller-runtime" | ||
"sigs.k8s.io/controller-runtime/pkg/client" | ||
|
||
"github.com/operator-framework/operator-controller/pkg/scheme" | ||
) | ||
|
||
const ( | ||
testClusterCatalogNameEnv = "TEST_CLUSTER_CATALOG_NAME" | ||
testClusterExtensionNameEnv = "TEST_CLUSTER_EXTENSION_NAME" | ||
) | ||
|
||
var ( | ||
c client.Client | ||
kclientset kubernetes.Interface | ||
|
||
testClusterCatalogName string | ||
testClusterExtensionName string | ||
) | ||
|
||
func TestMain(m *testing.M) { | ||
var ok bool | ||
testClusterCatalogName, ok = os.LookupEnv(testClusterCatalogNameEnv) | ||
if !ok { | ||
fmt.Printf("%q is not set", testClusterCatalogNameEnv) | ||
os.Exit(1) | ||
} | ||
testClusterExtensionName, ok = os.LookupEnv(testClusterExtensionNameEnv) | ||
if !ok { | ||
fmt.Printf("%q is not set", testClusterExtensionNameEnv) | ||
os.Exit(1) | ||
} | ||
|
||
cfg := ctrl.GetConfigOrDie() | ||
|
||
var err error | ||
c, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) | ||
if err != nil { | ||
fmt.Printf("failed to create client: %s\n", err) | ||
os.Exit(1) | ||
} | ||
|
||
kclientset, err = kubernetes.NewForConfig(ctrl.GetConfigOrDie()) | ||
if err != nil { | ||
fmt.Printf("failed to create kubernetes clientset: %s\n", err) | ||
os.Exit(1) | ||
} | ||
|
||
os.Exit(m.Run()) | ||
} |