From 6c05cd6805fd867966b02ae44f385f5069219a35 Mon Sep 17 00:00:00 2001 From: Tomas Remes Date: Wed, 3 Mar 2021 18:21:53 +0100 Subject: [PATCH 1/2] [release-4.7] Bug 1935070: Extend the OLM operator data with related ClusterServiceVersion conditions (#358) * Extend the OLM operator data with related ClusterServiceVersion conditions * Update sample archive * Update the OLM operators test * Fix --- docs/gathered-data.md | 6 +- .../config/olm_operators.json | 199 +++++++++++++++++- pkg/gather/clusterconfig/olm_operators.go | 68 ++++-- .../clusterconfig/olm_operators_test.go | 58 +++-- pkg/gather/clusterconfig/testdata/csv_1.yaml | 17 ++ 5 files changed, 311 insertions(+), 37 deletions(-) create mode 100644 pkg/gather/clusterconfig/testdata/csv_1.yaml diff --git a/docs/gathered-data.md b/docs/gathered-data.md index 25015fae5..ddff3b2fd 100644 --- a/docs/gathered-data.md +++ b/docs/gathered-data.md @@ -311,7 +311,11 @@ Output raw size: 491 ## OLMOperators -collects list of all names (including version) of installed OLM operators. +collects list of installed OLM operators. +Each OLM operator (in the list) contains following data: +- OLM operator name +- OLM operator version +- related ClusterServiceVersion conditions See: docs/insights-archive-sample/config/olm_operators Location of in archive: config/olm_operators diff --git a/docs/insights-archive-sample/config/olm_operators.json b/docs/insights-archive-sample/config/olm_operators.json index 64bbbe03a..0911d187e 100644 --- a/docs/insights-archive-sample/config/olm_operators.json +++ b/docs/insights-archive-sample/config/olm_operators.json @@ -1,22 +1,207 @@ [ { "name": "eap.openshift-operators", - "version": "v2.0.2" + "version": "v2.1.1", + "csv_conditions": [ + { + "lastTransitionTime": "2021-03-02T08:52:24Z", + "lastUpdateTime": "2021-03-02T08:52:24Z", + "message": "requirements not yet checked", + "phase": "Pending", + "reason": "RequirementsUnknown" + }, + { + "lastTransitionTime": "2021-03-02T08:52:24Z", + "lastUpdateTime": "2021-03-02T08:52:24Z", + "message": "all requirements found, attempting install", + "phase": "InstallReady", + "reason": "AllRequirementsMet" + }, + { + "lastTransitionTime": "2021-03-02T08:52:24Z", + "lastUpdateTime": "2021-03-02T08:52:24Z", + "message": "waiting for install components to report healthy", + "phase": "Installing", + "reason": "InstallSucceeded" + }, + { + "lastTransitionTime": "2021-03-02T08:52:24Z", + "lastUpdateTime": "2021-03-02T08:52:25Z", + "message": "installing: waiting for deployment eap-operator to become ready: Waiting for rollout to finish: 0 of 1 updated replicas are available...\n", + "phase": "Installing", + "reason": "InstallWaiting" + }, + { + "lastTransitionTime": "2021-03-02T08:52:46Z", + "lastUpdateTime": "2021-03-02T08:52:46Z", + "message": "install strategy completed with no errors", + "phase": "Succeeded", + "reason": "InstallSucceeded" + } + ] }, { "name": "elasticsearch-operator.openshift-operators-redhat", - "version": "4.6.0-202011221454.p0" + "version": "4.6.0-202102200141.p0", + "csv_conditions": [ + { + "lastTransitionTime": "2021-03-02T06:38:14Z", + "lastUpdateTime": "2021-03-02T06:38:14Z", + "message": "requirements not yet checked", + "phase": "Pending", + "reason": "RequirementsUnknown" + }, + { + "lastTransitionTime": "2021-03-02T06:38:15Z", + "lastUpdateTime": "2021-03-02T06:38:15Z", + "message": "all requirements found, attempting install", + "phase": "InstallReady", + "reason": "AllRequirementsMet" + }, + { + "lastTransitionTime": "2021-03-02T06:38:16Z", + "lastUpdateTime": "2021-03-02T06:38:16Z", + "message": "waiting for install components to report healthy", + "phase": "Installing", + "reason": "InstallSucceeded" + }, + { + "lastTransitionTime": "2021-03-02T06:38:16Z", + "lastUpdateTime": "2021-03-02T06:38:17Z", + "message": "installing: waiting for deployment elasticsearch-operator to become ready: Waiting for rollout to finish: 1 old replicas are pending termination...\n", + "phase": "Installing", + "reason": "InstallWaiting" + }, + { + "lastTransitionTime": "2021-03-02T06:38:26Z", + "lastUpdateTime": "2021-03-02T06:38:26Z", + "message": "install strategy completed with no errors", + "phase": "Succeeded", + "reason": "InstallSucceeded" + } + ] }, { "name": "kiali-ossm.openshift-operators", - "version": "v1.24.4" + "version": "v1.24.5", + "csv_conditions": [ + { + "lastTransitionTime": "2021-03-02T08:52:09Z", + "lastUpdateTime": "2021-03-02T08:52:09Z", + "message": "requirements not yet checked", + "phase": "Pending", + "reason": "RequirementsUnknown" + }, + { + "lastTransitionTime": "2021-03-02T08:52:09Z", + "lastUpdateTime": "2021-03-02T08:52:09Z", + "message": "all requirements found, attempting install", + "phase": "InstallReady", + "reason": "AllRequirementsMet" + }, + { + "lastTransitionTime": "2021-03-02T08:52:09Z", + "lastUpdateTime": "2021-03-02T08:52:09Z", + "message": "waiting for install components to report healthy", + "phase": "Installing", + "reason": "InstallSucceeded" + }, + { + "lastTransitionTime": "2021-03-02T08:52:09Z", + "lastUpdateTime": "2021-03-02T08:52:10Z", + "message": "installing: waiting for deployment kiali-operator to become ready: Waiting for rollout to finish: 0 of 1 updated replicas are available...\n", + "phase": "Installing", + "reason": "InstallWaiting" + }, + { + "lastTransitionTime": "2021-03-02T08:52:29Z", + "lastUpdateTime": "2021-03-02T08:52:29Z", + "message": "install strategy completed with no errors", + "phase": "Succeeded", + "reason": "InstallSucceeded" + } + ] }, { - "name": "postgresql-operator-dev4devs-com.postgresql-operator", - "version": "v0.1.1" + "name": "postgresql-operator-dev4devs-com.psql-test", + "version": "v0.1.1", + "csv_conditions": [ + { + "lastTransitionTime": "2021-03-02T08:53:34Z", + "lastUpdateTime": "2021-03-02T08:53:34Z", + "message": "requirements not yet checked", + "phase": "Pending", + "reason": "RequirementsUnknown" + }, + { + "lastTransitionTime": "2021-03-02T08:53:35Z", + "lastUpdateTime": "2021-03-02T08:53:35Z", + "message": "all requirements found, attempting install", + "phase": "InstallReady", + "reason": "AllRequirementsMet" + }, + { + "lastTransitionTime": "2021-03-02T08:53:35Z", + "lastUpdateTime": "2021-03-02T08:53:35Z", + "message": "waiting for install components to report healthy", + "phase": "Installing", + "reason": "InstallSucceeded" + }, + { + "lastTransitionTime": "2021-03-02T08:53:35Z", + "lastUpdateTime": "2021-03-02T08:53:36Z", + "message": "installing: waiting for deployment postgresql-operator to become ready: Waiting for rollout to finish: 0 of 1 updated replicas are available...\n", + "phase": "Installing", + "reason": "InstallWaiting" + }, + { + "lastTransitionTime": "2021-03-02T08:53:47Z", + "lastUpdateTime": "2021-03-02T08:53:47Z", + "message": "install strategy completed with no errors", + "phase": "Succeeded", + "reason": "InstallSucceeded" + } + ] }, { "name": "radanalytics-spark.openshift-operators", - "version": "v1.1.0" + "version": "v1.1.0", + "csv_conditions": [ + { + "lastTransitionTime": "2021-03-02T08:55:36Z", + "lastUpdateTime": "2021-03-02T08:55:36Z", + "message": "requirements not yet checked", + "phase": "Pending", + "reason": "RequirementsUnknown" + }, + { + "lastTransitionTime": "2021-03-02T08:55:37Z", + "lastUpdateTime": "2021-03-02T08:55:37Z", + "message": "all requirements found, attempting install", + "phase": "InstallReady", + "reason": "AllRequirementsMet" + }, + { + "lastTransitionTime": "2021-03-02T08:55:38Z", + "lastUpdateTime": "2021-03-02T08:55:38Z", + "message": "waiting for install components to report healthy", + "phase": "Installing", + "reason": "InstallSucceeded" + }, + { + "lastTransitionTime": "2021-03-02T08:55:38Z", + "lastUpdateTime": "2021-03-02T08:55:38Z", + "message": "installing: waiting for deployment spark-operator to become ready: Waiting for rollout to finish: 0 of 1 updated replicas are available...\n", + "phase": "Installing", + "reason": "InstallWaiting" + }, + { + "lastTransitionTime": "2021-03-02T08:55:51Z", + "lastUpdateTime": "2021-03-02T08:55:51Z", + "message": "install strategy completed with no errors", + "phase": "Succeeded", + "reason": "InstallSucceeded" + } + ] } -] \ No newline at end of file +] diff --git a/pkg/gather/clusterconfig/olm_operators.go b/pkg/gather/clusterconfig/olm_operators.go index 4feafb4c8..cd5adf6dd 100644 --- a/pkg/gather/clusterconfig/olm_operators.go +++ b/pkg/gather/clusterconfig/olm_operators.go @@ -15,12 +15,29 @@ import ( "github.com/openshift/insights-operator/pkg/record" ) +var ( + operatorGVR = schema.GroupVersionResource{Group: "operators.coreos.com", Version: "v1", Resource: "operators"} + clusterServiceVersionGVR = schema.GroupVersionResource{Group: "operators.coreos.com", Version: "v1alpha1", Resource: "clusterserviceversions"} +) + type olmOperator struct { - Name string `json:"name"` - Version string `json:"version"` + Name string `json:"name"` + Version string `json:"version"` + Conditions []interface{} `json:"csv_conditions"` +} + +// ClusterServiceVersion helper struct +type csvRef struct { + Name string + Namespace string + Version string } -// GatherOLMOperators collects list of all names (including version) of installed OLM operators. +// GatherOLMOperators collects list of installed OLM operators. +// Each OLM operator (in the list) contains following data: +// - OLM operator name +// - OLM operator version +// - related ClusterServiceVersion conditions // // See: docs/insights-archive-sample/config/olm_operators // Location of in archive: config/olm_operators @@ -37,8 +54,7 @@ func GatherOLMOperators(g *Gatherer, c chan<- gatherResult) { } func gatherOLMOperators(ctx context.Context, dynamicClient dynamic.Interface) ([]record.Record, []error) { - gvr := schema.GroupVersionResource{Group: "operators.coreos.com", Version: "v1", Resource: "operators"} - olmOperators, err := dynamicClient.Resource(gvr).List(ctx, metav1.ListOptions{}) + olmOperators, err := dynamicClient.Resource(operatorGVR).List(ctx, metav1.ListOptions{}) if errors.IsNotFound(err) { return nil, nil } @@ -54,13 +70,19 @@ func gatherOLMOperators(ctx context.Context, dynamicClient dynamic.Interface) ([ continue } for _, r := range refs { - ver := readVersionFromRefs(r) - if ver == "" { + csvRef := getCSVRefFromRefs(r) + if csvRef == nil { + continue + } + conditions, err := getCSVConditions(ctx, dynamicClient, csvRef) + if err != nil { + klog.Errorf("failed to get %s conditions: %v", csvRef.Name, err) continue } olmO := olmOperator{ - Name: i.GetName(), - Version: ver, + Name: i.GetName(), + Version: csvRef.Version, + Conditions: conditions, } if isInArray(olmO, olms) { continue @@ -80,26 +102,44 @@ func gatherOLMOperators(ctx context.Context, dynamicClient dynamic.Interface) ([ func isInArray(o olmOperator, a []olmOperator) bool { for _, op := range a { - if o == op { + if o.Name == op.Name && o.Version == op.Version { return true } } return false } -func readVersionFromRefs(r interface{}) string { +func getCSVRefFromRefs(r interface{}) *csvRef { refMap, ok := r.(map[string]interface{}) if !ok { klog.Errorf("Cannot convert %s to map[string]interface{}", r) - return "" + return nil } // version is part of the name of ClusterServiceVersion if refMap["kind"] == "ClusterServiceVersion" { name := refMap["name"].(string) nameVer := strings.SplitN(name, ".", 2) - return nameVer[1] + csvRef := &csvRef{ + Name: name, + Namespace: refMap["namespace"].(string), + Version: nameVer[1], + } + return csvRef + } + return nil +} + +func getCSVConditions(ctx context.Context, dynamicClient dynamic.Interface, csvRef *csvRef) ([]interface{}, error) { + csv, err := dynamicClient.Resource(clusterServiceVersionGVR).Namespace(csvRef.Namespace).Get(ctx, csvRef.Name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + var conditions []interface{} + err = parseJSONQuery(csv.Object, "status.conditions", &conditions) + if err != nil { + return nil, err } - return "" + return conditions, nil } // OlmOperatorAnonymizer implements HostSubnet serialization diff --git a/pkg/gather/clusterconfig/olm_operators_test.go b/pkg/gather/clusterconfig/olm_operators_test.go index 603e7361f..1efcfb9e2 100644 --- a/pkg/gather/clusterconfig/olm_operators_test.go +++ b/pkg/gather/clusterconfig/olm_operators_test.go @@ -15,29 +15,26 @@ import ( ) func TestOLMOperatorsGather(t *testing.T) { - f, err := os.Open("testdata/olm_operator_1.yaml") + olmOpContent, err := readFromFile("testdata/olm_operator_1.yaml") if err != nil { t.Fatal("test failed to read OLM operator data", err) } - olmOpContent, err := ioutil.ReadAll(f) + + csvContent, err := readFromFile("testdata/csv_1.yaml") if err != nil { - t.Fatal("error reading test data file", err) + t.Fatal("test failed to read CSV ", err) } - gvr := schema.GroupVersionResource{Group: "operators.coreos.com", Version: "v1", Resource: "operators"} client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ - gvr: "OperatorsList", + operatorGVR: "OperatorsList", + clusterServiceVersionGVR: "ClusterServiceVersionsList", }) - decUnstructured := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) - - testOLMOperator := &unstructured.Unstructured{} - - _, _, err = decUnstructured.Decode(olmOpContent, nil, testOLMOperator) + err = createUnstructuredResource(olmOpContent, client, operatorGVR) if err != nil { - t.Fatal("unable to decode OLM operator ", err) + t.Fatal("cannot create OLM operator ", err) } - _, err = client.Resource(gvr).Create(context.Background(), testOLMOperator, metav1.CreateOptions{}) + err = createUnstructuredResource(csvContent, client, clusterServiceVersionGVR) if err != nil { - t.Fatal("unable to create fake OLM operator ", err) + t.Fatal("cannot create ClusterServiceVersion ", err) } ctx := context.Background() @@ -54,9 +51,40 @@ func TestOLMOperatorsGather(t *testing.T) { t.Fatalf("returned item is not of type OlmOperatorAnonymizer") } if ooa.operators[0].Name != "test-olm-operator" { - t.Fatalf("unexpected name of gathered OLM operator %s", ooa.operators[0]) + t.Fatalf("unexpected name of gathered OLM operator %s", ooa.operators[0].Name) } if ooa.operators[0].Version != "v1.2.3" { - t.Fatalf("unexpected version of gathered OLM operator %s", ooa.operators[0]) + t.Fatalf("unexpected version of gathered OLM operator %s", ooa.operators[0].Version) + } + if len(ooa.operators[0].Conditions) != 2 { + t.Fatalf("unexpected number of conditions %s", ooa.operators[0].Conditions...) + } +} + +func createUnstructuredResource(content []byte, client *dynamicfake.FakeDynamicClient, gvr schema.GroupVersionResource) error { + decUnstructured := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) + unstructuredResource := &unstructured.Unstructured{} + + _, _, err := decUnstructured.Decode(content, nil, unstructuredResource) + if err != nil { + return err + } + + _, err = client.Resource(gvr).Namespace("test-olm-operator").Create(context.Background(), unstructuredResource, metav1.CreateOptions{}) + if err != nil { + return err + } + return nil +} + +func readFromFile(filePath string) ([]byte, error) { + f, err := os.Open(filePath) + if err != nil { + return nil, err + } + content, err := ioutil.ReadAll(f) + if err != nil { + return nil, err } + return content, nil } diff --git a/pkg/gather/clusterconfig/testdata/csv_1.yaml b/pkg/gather/clusterconfig/testdata/csv_1.yaml new file mode 100644 index 000000000..0ca8f5502 --- /dev/null +++ b/pkg/gather/clusterconfig/testdata/csv_1.yaml @@ -0,0 +1,17 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + name: test-olm-operator.v1.2.3 + namespace: test-olm-operator +status: + conditions: + - lastTransitionTime: "2021-03-02T08:52:24Z" + lastUpdateTime: "2021-03-02T08:52:24Z" + message: requirements not yet checked + phase: Pending + reason: RequirementsUnknown + - lastTransitionTime: "2021-03-02T08:52:24Z" + lastUpdateTime: "2021-03-02T08:52:24Z" + message: all requirements found, attempting install + phase: InstallReady + reason: AllRequirementsMet From 6c2e69421f287291b9685828680324c2ce926487 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Reme=C5=A1?= Date: Tue, 9 Mar 2021 08:59:15 +0100 Subject: [PATCH 2/2] Test fix --- pkg/gather/clusterconfig/olm_operators_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/gather/clusterconfig/olm_operators_test.go b/pkg/gather/clusterconfig/olm_operators_test.go index 1efcfb9e2..48afaf834 100644 --- a/pkg/gather/clusterconfig/olm_operators_test.go +++ b/pkg/gather/clusterconfig/olm_operators_test.go @@ -79,6 +79,7 @@ func createUnstructuredResource(content []byte, client *dynamicfake.FakeDynamicC func readFromFile(filePath string) ([]byte, error) { f, err := os.Open(filePath) + defer f.Close() if err != nil { return nil, err }