diff --git a/docs/gathered-data.md b/docs/gathered-data.md index 1bd034bf7..15e0c0602 100644 --- a/docs/gathered-data.md +++ b/docs/gathered-data.md @@ -684,6 +684,17 @@ API Reference: * 4.9+ +## OpenshiftMachineAPIEvents + +collects warning ("abnormal") events +from "openshift-machine-api" namespace + +* Location of events in archive: events/ +* Id in config: clusterconfig/openshift_machine_api_events +* Since versions: + * 4.12+ + + ## OpenshiftSDNControllerLogs collects logs from sdn-controller pod in openshift-sdn namespace with following substrings: diff --git a/docs/insights-archive-sample/events/openshift-machine-api.json b/docs/insights-archive-sample/events/openshift-machine-api.json new file mode 100644 index 000000000..6eb4b5a68 --- /dev/null +++ b/docs/insights-archive-sample/events/openshift-machine-api.json @@ -0,0 +1,18 @@ +{ + "items":[ + { + "namespace":"openshift-machine-api", + "lastTimestamp":"2022-08-24T11:40:44+02:00", + "reason":"FailedUpdate", + "message":"cluster-lrqft-worker-us-east-2a-jsmg4: reconciler failed to Update machine: requeue in: 20s", + "type":"Warning" + }, + { + "namespace":"openshift-machine-api", + "lastTimestamp":"2022-08-24T11:41:05+02:00", + "reason":"FailedUpdate", + "message":"cluster-lrqft-worker-us-east-2c-rlfpx: reconciler failed to Update machine: requeue in: 20s", + "type":"Warning" + } + ] +} \ No newline at end of file diff --git a/pkg/gatherers/clusterconfig/clusterconfig_gatherer.go b/pkg/gatherers/clusterconfig/clusterconfig_gatherer.go index c2755ea26..bac5a2b00 100644 --- a/pkg/gatherers/clusterconfig/clusterconfig_gatherer.go +++ b/pkg/gatherers/clusterconfig/clusterconfig_gatherer.go @@ -84,6 +84,7 @@ var gatheringFunctions = map[string]gathererFuncPtr{ "support_secret": (*Gatherer).GatherSupportSecret, "active_alerts": (*Gatherer).GatherActiveAlerts, "ceph_cluster": (*Gatherer).GatherCephCluster, + "openshift_machine_api_events": (*Gatherer).GatherOpenshiftMachineAPIEvents, } func New( diff --git a/pkg/gatherers/clusterconfig/events_filtering.go b/pkg/gatherers/clusterconfig/events_filtering.go new file mode 100644 index 000000000..0fd64ad4d --- /dev/null +++ b/pkg/gatherers/clusterconfig/events_filtering.go @@ -0,0 +1,77 @@ +package clusterconfig + +import ( + "sort" + "time" + + v1 "k8s.io/api/core/v1" +) + +// getEventsForInterval() returns events that occoured since last interval +func getEventsForInterval(interval time.Duration, events *v1.EventList) v1.EventList { + oldestEventTime := time.Now().Add(-interval) + var filteredEvents v1.EventList + for i := range events.Items { + if isEventNew(&events.Items[i], oldestEventTime) { + filteredEvents.Items = append(filteredEvents.Items, events.Items[i]) + } + } + return filteredEvents +} + +// isEventNew() returns true if event occoured after given time, otherwise returns false +func isEventNew(event *v1.Event, oldestEventTime time.Time) bool { + if event.LastTimestamp.Time.After(oldestEventTime) { + return true + // if LastTimestamp is zero then try to check the event series + } else if event.LastTimestamp.IsZero() { + if event.Series != nil { + if event.Series.LastObservedTime.Time.After(oldestEventTime) { + return true + } + } + } + return false +} + +// filterAbnormalEvents returns events that have Type different from "Normal" +func filterAbnormalEvents(events *v1.EventList) v1.EventList { + var filteredEvents v1.EventList + for i := range events.Items { + if isEventAbnormal(&events.Items[i]) { + filteredEvents.Items = append(filteredEvents.Items, events.Items[i]) + } + } + return filteredEvents +} + +func isEventAbnormal(event *v1.Event) bool { + return event.Type != "Normal" +} + +// eventListToCompactedEventList() converts EventList into CompactedEventList +func eventListToCompactedEventList(events *v1.EventList) CompactedEventList { + var compactedEvents CompactedEventList + for i := range events.Items { + event := events.Items[i] + compactedEvent := CompactedEvent{ + Namespace: event.Namespace, + LastTimestamp: event.LastTimestamp.Time, + Reason: event.Reason, + Message: event.Message, + Type: event.Type, + } + if event.LastTimestamp.Time.IsZero() { + if event.Series != nil { + compactedEvent.LastTimestamp = event.Series.LastObservedTime.Time + } + } + compactedEvents.Items = append(compactedEvents.Items, compactedEvent) + } + + sort.Slice(compactedEvents.Items, func(i, j int) bool { + return compactedEvents.Items[i].LastTimestamp.Before(compactedEvents.Items[j].LastTimestamp) + }) + + return compactedEvents +} diff --git a/pkg/gatherers/clusterconfig/events_filtering_test.go b/pkg/gatherers/clusterconfig/events_filtering_test.go new file mode 100644 index 000000000..0f5be3afc --- /dev/null +++ b/pkg/gatherers/clusterconfig/events_filtering_test.go @@ -0,0 +1,162 @@ +package clusterconfig + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func Test_getEventsForInterval(t *testing.T) { + timeNow := time.Now() + test := struct { + events v1.EventList + expected v1.EventList + }{ + events: v1.EventList{ + Items: []v1.Event{ + { + ObjectMeta: metav1.ObjectMeta{Name: "oldEvent1"}, + LastTimestamp: metav1.Time{}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "newEvent1"}, + LastTimestamp: metav1.NewTime(timeNow), + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "oldEvent2"}, + LastTimestamp: metav1.Time{}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "newEvent2"}, + LastTimestamp: metav1.NewTime(timeNow), + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "newEvent3"}, + LastTimestamp: metav1.NewTime(timeNow), + }, + }, + }, + expected: v1.EventList{ + Items: []v1.Event{ + { + ObjectMeta: metav1.ObjectMeta{Name: "newEvent1"}, + LastTimestamp: metav1.NewTime(timeNow), + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "newEvent2"}, + LastTimestamp: metav1.NewTime(timeNow), + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "newEvent3"}, + LastTimestamp: metav1.NewTime(timeNow), + }, + }, + }, + } + + filteredEvents := getEventsForInterval(1*time.Minute, &test.events) + assert.Equal(t, filteredEvents, test.expected) +} + +func Test_filterAbnormalEvents(t *testing.T) { + test := struct { + events v1.EventList + expected v1.EventList + }{ + events: v1.EventList{ + Items: []v1.Event{ + { + ObjectMeta: metav1.ObjectMeta{Name: "normalEvent1"}, + Type: "Normal", + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "warningEvent1"}, + Type: "Warning", + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "normalEvent2"}, + Type: "Normal", + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "warningEvent2"}, + Type: "Warning", + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "warningEvent3"}, + Type: "Warning", + }, + }, + }, + expected: v1.EventList{ + Items: []v1.Event{ + { + ObjectMeta: metav1.ObjectMeta{Name: "warningEvent1"}, + Type: "Warning", + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "warningEvent2"}, + Type: "Warning", + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "warningEvent3"}, + Type: "Warning", + }, + }, + }, + } + + filteredEvents := filterAbnormalEvents(&test.events) + assert.Equal(t, filteredEvents, test.expected) +} + +func Test_isEventNew(t *testing.T) { + tests := []struct { + event v1.Event + expected bool + }{ + { + event: v1.Event{ + ObjectMeta: metav1.ObjectMeta{Name: "newEvent"}, + LastTimestamp: metav1.Now(), + Type: "Normal", + }, + expected: true, + }, + { + event: v1.Event{ + ObjectMeta: metav1.ObjectMeta{Name: "oldEvent"}, + LastTimestamp: metav1.NewTime(time.Now().Add(-6 * time.Minute)), + Type: "Normal", + }, + expected: false, + }, + } + + for _, test := range tests { + assert.Equal(t, isEventNew(&test.event, time.Now().Add(-5*time.Minute)), test.expected) + } +} + +func Test_eventListToCompactedEventList(t *testing.T) { + timeNow := time.Now() + event := v1.Event{ + ObjectMeta: metav1.ObjectMeta{Name: "event", Namespace: "test namespace"}, + LastTimestamp: metav1.NewTime(timeNow), + Type: "Normal", + Reason: "test reason", + Message: "test message", + } + compactedEvent := CompactedEvent{ + Namespace: "test namespace", + LastTimestamp: timeNow, + Reason: "test reason", + Message: "test message", + Type: "Normal", + } + compactedEventList := eventListToCompactedEventList(&v1.EventList{Items: []v1.Event{event}}) + + assert.Equal(t, compactedEvent, compactedEventList.Items[0]) +} diff --git a/pkg/gatherers/clusterconfig/openshift_machine_api_events.go b/pkg/gatherers/clusterconfig/openshift_machine_api_events.go new file mode 100644 index 000000000..6091b2912 --- /dev/null +++ b/pkg/gatherers/clusterconfig/openshift_machine_api_events.go @@ -0,0 +1,49 @@ +package clusterconfig + +import ( + "context" + "time" + + "github.com/openshift/insights-operator/pkg/record" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" +) + +// GatherOpenshiftMachineAPIEvents collects warning ("abnormal") events +// from "openshift-machine-api" namespace +// +// * Location of events in archive: events/ +// * Id in config: clusterconfig/openshift_machine_api_events +// * Since versions: +// - 4.12+ +func (g *Gatherer) GatherOpenshiftMachineAPIEvents(ctx context.Context) ([]record.Record, []error) { + gatherKubeClient, err := kubernetes.NewForConfig(g.gatherProtoKubeConfig) + if err != nil { + return nil, []error{err} + } + records, err := gatherOpenshiftMachineAPIEvents(ctx, gatherKubeClient.CoreV1(), g.interval) + if err != nil { + return nil, []error{err} + } + return records, nil +} + +func gatherOpenshiftMachineAPIEvents(ctx context.Context, + coreClient corev1client.CoreV1Interface, + interval time.Duration) ([]record.Record, error) { + events, err := coreClient.Events("openshift-machine-api").List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + // filter the event list to only recent events with type different than "Normal" + filteredEvents := getEventsForInterval(interval, events) + filteredEvents = filterAbnormalEvents(&filteredEvents) + + if len(filteredEvents.Items) == 0 { + return nil, nil + } + compactedEvents := eventListToCompactedEventList(&filteredEvents) + + return []record.Record{{Name: "events/openshift-machine-api", Item: record.JSONMarshaller{Object: &compactedEvents}}}, nil +} diff --git a/pkg/gatherers/clusterconfig/openshift_machine_api_events_test.go b/pkg/gatherers/clusterconfig/openshift_machine_api_events_test.go new file mode 100644 index 000000000..e935461a8 --- /dev/null +++ b/pkg/gatherers/clusterconfig/openshift_machine_api_events_test.go @@ -0,0 +1,82 @@ +package clusterconfig + +import ( + "context" + "testing" + "time" + + "github.com/openshift/insights-operator/pkg/record" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubefake "k8s.io/client-go/kubernetes/fake" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" +) + +func Test_WarningEvents_gatherOpenshiftMachineAPIEvents(t *testing.T) { + normalEvent1 := v1.Event{ + ObjectMeta: metav1.ObjectMeta{Name: "normalEvent1", Namespace: "openshift-machine-api"}, + LastTimestamp: metav1.Now(), + Type: "Normal", + Reason: "normal", + } + warningEvent1 := v1.Event{ + ObjectMeta: metav1.ObjectMeta{Name: "warningEvent1", Namespace: "openshift-machine-api"}, + LastTimestamp: metav1.Now(), + Type: "Warning", + Reason: "warning", + } + normalEvent2 := v1.Event{ + ObjectMeta: metav1.ObjectMeta{Name: "normalEvent2", Namespace: "openshift-machine-api"}, + LastTimestamp: metav1.Now(), + Type: "Normal", + Reason: "normal", + } + warningEvent2 := v1.Event{ + ObjectMeta: metav1.ObjectMeta{Name: "warningEvent2", Namespace: "openshift-machine-api"}, + LastTimestamp: metav1.Now(), + Type: "Warning", + Reason: "warning", + } + warningEvent3 := v1.Event{ + ObjectMeta: metav1.ObjectMeta{Name: "warningEvent3", Namespace: "openshift-machine-api"}, + LastTimestamp: metav1.Time{}, + Type: "Warning", + Reason: "warning", + } + normalEvent3 := v1.Event{ + ObjectMeta: metav1.ObjectMeta{Name: "normalEvent3", Namespace: "openshift-machine-api"}, + LastTimestamp: metav1.Time{}, + Type: "Normal", + Reason: "normal", + } + var events v1.EventList + events.Items = append(events.Items, warningEvent1, warningEvent2) + compactedEvents := eventListToCompactedEventList(&events) + + type args struct { + ctx context.Context + coreClient corev1client.CoreV1Interface + } + test := struct { + name string + args args + wantErr bool + want []record.Record + }{ + name: "openshift-machine-api warning events", + args: args{ + ctx: context.TODO(), + coreClient: kubefake.NewSimpleClientset(&warningEvent1, &normalEvent1, &normalEvent2, + &warningEvent2, &warningEvent3, &normalEvent3).CoreV1(), + }, + wantErr: false, + want: []record.Record{{Name: "events/openshift-machine-api", Item: record.JSONMarshaller{Object: &compactedEvents}}}, + } + + t.Run(test.name, func(t *testing.T) { + got, err := gatherOpenshiftMachineAPIEvents(test.args.ctx, test.args.coreClient, 1*time.Minute) + assert.NoError(t, err) + assert.Equal(t, test.want, got) + }) +} diff --git a/pkg/gatherers/clusterconfig/operators_pods_and_events.go b/pkg/gatherers/clusterconfig/operators_pods_and_events.go index 7989da455..de853346a 100644 --- a/pkg/gatherers/clusterconfig/operators_pods_and_events.go +++ b/pkg/gatherers/clusterconfig/operators_pods_and_events.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "regexp" - "sort" "strings" "time" @@ -33,6 +32,7 @@ type CompactedEvent struct { LastTimestamp time.Time `json:"lastTimestamp"` Reason string `json:"reason"` Message string `json:"message"` + Type string `json:"type"` } // CompactedEventList is collection of events @@ -189,40 +189,11 @@ func gatherNamespaceEvents(ctx context.Context, return nil, err } // filter the event list to only recent events - oldestEventTime := time.Now().Add(-interval) - var filteredEventIndex []int - for i := range events.Items { - // if LastTimestamp is zero then try to check the event series - if events.Items[i].LastTimestamp.IsZero() { - if events.Items[i].Series != nil { - if events.Items[i].Series.LastObservedTime.Time.After(oldestEventTime) { - filteredEventIndex = append(filteredEventIndex, i) - } - } - } else { - if events.Items[i].LastTimestamp.Time.After(oldestEventTime) { - filteredEventIndex = append(filteredEventIndex, i) - } - } - } - if len(filteredEventIndex) == 0 { + filteredEvents := getEventsForInterval(interval, events) + if len(filteredEvents.Items) == 0 { return nil, nil } - compactedEvents := CompactedEventList{Items: make([]CompactedEvent, len(filteredEventIndex))} - for i, index := range filteredEventIndex { - compactedEvents.Items[i] = CompactedEvent{ - Namespace: events.Items[index].Namespace, - LastTimestamp: events.Items[index].LastTimestamp.Time, - Reason: events.Items[index].Reason, - Message: events.Items[index].Message, - } - if events.Items[index].LastTimestamp.Time.IsZero() { - compactedEvents.Items[i].LastTimestamp = events.Items[index].Series.LastObservedTime.Time - } - } - sort.Slice(compactedEvents.Items, func(i, j int) bool { - return compactedEvents.Items[i].LastTimestamp.Before(compactedEvents.Items[j].LastTimestamp) - }) + compactedEvents := eventListToCompactedEventList(&filteredEvents) return []record.Record{{Name: fmt.Sprintf("events/%s", namespace), Item: record.JSONMarshaller{Object: &compactedEvents}}}, nil }