From fbdbcc57ed863e2be42d3454b5353083ac31d42b Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Fri, 22 Nov 2024 18:37:17 +0000 Subject: [PATCH] Tail journald logs into report on suite failure Signed-off-by: Brad Davidson --- tests/e2e/dualstack/dualstack_test.go | 4 +++- tests/e2e/embeddedmirror/embeddedmirror_test.go | 4 +++- tests/e2e/externalip/externalip_test.go | 4 +++- tests/e2e/privateregistry/privateregistry_test.go | 5 +++-- tests/e2e/rootless/rootless_test.go | 4 +++- tests/e2e/rotateca/rotateca_test.go | 4 +++- tests/e2e/s3/s3_test.go | 5 +++-- .../secretsencryption/secretsencryption_test.go | 4 +++- tests/e2e/snapshotrestore/snapshotrestore_test.go | 4 +++- tests/e2e/splitserver/splitserver_test.go | 8 +++++--- tests/e2e/startup/startup_test.go | 4 +++- .../svcpoliciesandfirewall_test.go | 9 +++++---- tests/e2e/tailscale/tailscale_test.go | 4 +++- tests/e2e/testutils.go | 15 ++++++++++++++- tests/e2e/token/token_test.go | 4 +++- tests/e2e/upgradecluster/upgradecluster_test.go | 15 ++++++++------- tests/e2e/validatecluster/validatecluster_test.go | 4 +++- tests/e2e/wasm/wasm_test.go | 6 ++++-- 18 files changed, 75 insertions(+), 32 deletions(-) diff --git a/tests/e2e/dualstack/dualstack_test.go b/tests/e2e/dualstack/dualstack_test.go index c9612f9b7142..9262af922cea 100644 --- a/tests/e2e/dualstack/dualstack_test.go +++ b/tests/e2e/dualstack/dualstack_test.go @@ -195,7 +195,9 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if !failed { + if failed { + AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } if !failed || *ci { diff --git a/tests/e2e/embeddedmirror/embeddedmirror_test.go b/tests/e2e/embeddedmirror/embeddedmirror_test.go index 7188b552b988..089fb465277b 100644 --- a/tests/e2e/embeddedmirror/embeddedmirror_test.go +++ b/tests/e2e/embeddedmirror/embeddedmirror_test.go @@ -146,7 +146,9 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if !failed { + if failed { + AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } if !failed || *ci { diff --git a/tests/e2e/externalip/externalip_test.go b/tests/e2e/externalip/externalip_test.go index 524bb8340276..9d2150991924 100644 --- a/tests/e2e/externalip/externalip_test.go +++ b/tests/e2e/externalip/externalip_test.go @@ -165,7 +165,9 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if !failed { + if failed { + AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } if !failed || *ci { diff --git a/tests/e2e/privateregistry/privateregistry_test.go b/tests/e2e/privateregistry/privateregistry_test.go index b3d4f2f237f8..e7495468a89e 100644 --- a/tests/e2e/privateregistry/privateregistry_test.go +++ b/tests/e2e/privateregistry/privateregistry_test.go @@ -145,8 +145,9 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - - if !failed { + if failed { + AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } if !failed || *ci { diff --git a/tests/e2e/rootless/rootless_test.go b/tests/e2e/rootless/rootless_test.go index 361778c72db7..4a205934e3d5 100644 --- a/tests/e2e/rootless/rootless_test.go +++ b/tests/e2e/rootless/rootless_test.go @@ -167,7 +167,9 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if !failed { + if failed { + AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, serverNodeNames)) + } else { Expect(e2e.GetCoverageReport(serverNodeNames)).To(Succeed()) } if !failed || *ci { diff --git a/tests/e2e/rotateca/rotateca_test.go b/tests/e2e/rotateca/rotateca_test.go index c43ab4d10899..3a6f2b0ca14f 100644 --- a/tests/e2e/rotateca/rotateca_test.go +++ b/tests/e2e/rotateca/rotateca_test.go @@ -138,7 +138,9 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if !failed { + if failed { + AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } if !failed || *ci { diff --git a/tests/e2e/s3/s3_test.go b/tests/e2e/s3/s3_test.go index fc3be6a5fde4..b61824525934 100644 --- a/tests/e2e/s3/s3_test.go +++ b/tests/e2e/s3/s3_test.go @@ -175,8 +175,9 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - - if !failed { + if failed { + AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } if !failed || *ci { diff --git a/tests/e2e/secretsencryption/secretsencryption_test.go b/tests/e2e/secretsencryption/secretsencryption_test.go index 187dcedba2fc..763e2f0ba381 100644 --- a/tests/e2e/secretsencryption/secretsencryption_test.go +++ b/tests/e2e/secretsencryption/secretsencryption_test.go @@ -221,7 +221,9 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if !failed { + if failed { + AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, serverNodeNames)) + } else { Expect(e2e.GetCoverageReport(serverNodeNames)).To(Succeed()) } if !failed || *ci { diff --git a/tests/e2e/snapshotrestore/snapshotrestore_test.go b/tests/e2e/snapshotrestore/snapshotrestore_test.go index 20deef629e74..f9ca105cb24b 100644 --- a/tests/e2e/snapshotrestore/snapshotrestore_test.go +++ b/tests/e2e/snapshotrestore/snapshotrestore_test.go @@ -317,7 +317,9 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if !failed { + if failed { + AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } if !failed || *ci { diff --git a/tests/e2e/splitserver/splitserver_test.go b/tests/e2e/splitserver/splitserver_test.go index c78520d67b41..642dbc1592e3 100644 --- a/tests/e2e/splitserver/splitserver_test.go +++ b/tests/e2e/splitserver/splitserver_test.go @@ -283,9 +283,11 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if !failed { - allNodes := append(cpNodeNames, etcdNodeNames...) - allNodes = append(allNodes, agentNodeNames...) + allNodes := append(cpNodeNames, etcdNodeNames...) + allNodes = append(allNodes, agentNodeNames...) + if failed { + AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, allNodes)) + } else { Expect(e2e.GetCoverageReport(allNodes)).To(Succeed()) } if !failed || *ci { diff --git a/tests/e2e/startup/startup_test.go b/tests/e2e/startup/startup_test.go index c926164fac14..fd1d71872490 100644 --- a/tests/e2e/startup/startup_test.go +++ b/tests/e2e/startup/startup_test.go @@ -310,7 +310,9 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if !failed { + if failed { + AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } if !failed || *ci { diff --git a/tests/e2e/svcpoliciesandfirewall/svcpoliciesandfirewall_test.go b/tests/e2e/svcpoliciesandfirewall/svcpoliciesandfirewall_test.go index 53128947a234..dec419e176c4 100644 --- a/tests/e2e/svcpoliciesandfirewall/svcpoliciesandfirewall_test.go +++ b/tests/e2e/svcpoliciesandfirewall/svcpoliciesandfirewall_test.go @@ -128,7 +128,7 @@ var _ = Describe("Verify Services Traffic policies and firewall config", Ordered Eventually(func(g Gomega) { externalIPs, _ := e2e.FetchExternalIPs(kubeConfigFile, lbSvcExt) g.Expect(externalIPs).To(HaveLen(1), "more than 1 exernalIP found") - g.Expect(externalIPs[0]).To(Equal(serverNodeIP),"external IP does not match servernodeIP") + g.Expect(externalIPs[0]).To(Equal(serverNodeIP), "external IP does not match servernodeIP") }, "25s", "5s").Should(Succeed()) }) @@ -154,7 +154,6 @@ var _ = Describe("Verify Services Traffic policies and firewall config", Ordered return e2e.RunCommand(cmd) }, "25s", "5s").ShouldNot(ContainSubstring("10.42")) - // Verify connectivity to the other nodeIP does not work because of external traffic policy=local for _, externalIP := range lbSvcExternalIPs { if externalIP == lbSvcExtExternalIPs[0] { @@ -250,7 +249,7 @@ var _ = Describe("Verify Services Traffic policies and firewall config", Ordered )) // Check the non working command fails because of internal traffic policy=local - Eventually(func() (bool) { + Eventually(func() bool { _, err := e2e.RunCommand(nonWorkingCmd) if err != nil && strings.Contains(err.Error(), "exit status") { // Treat exit status as a successful condition @@ -348,7 +347,9 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if !failed { + if failed { + AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } if !failed || *ci { diff --git a/tests/e2e/tailscale/tailscale_test.go b/tests/e2e/tailscale/tailscale_test.go index 3def1ac41ab5..449840e4f990 100644 --- a/tests/e2e/tailscale/tailscale_test.go +++ b/tests/e2e/tailscale/tailscale_test.go @@ -118,7 +118,9 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if !failed { + if failed { + AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } if !failed || *ci { diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index 950a3c8af896..29edcc7f951d 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -48,7 +48,7 @@ type NodeError struct { type SvcExternalIP struct { IP string `json:"ip"` - ipMode string `json:"ipMode"` + IPMode string `json:"ipMode"` } type ObjIP struct { @@ -364,6 +364,19 @@ func GetJournalLogs(node string) (string, error) { return RunCmdOnNode(cmd, node) } +func TailJournalLogs(lines int, nodes []string) string { + logs := &strings.Builder{} + for _, node := range nodes { + cmd := fmt.Sprintf("journalctl -u k3s* --no-pager --lines=%d", lines) + if l, err := RunCmdOnNode(cmd, node); err != nil { + fmt.Fprintf(logs, "** failed to read journald log for node %s ***\n%v\n", node, err) + } else { + fmt.Fprintf(logs, "** journald log for node %s ***\n%s\n", node, l) + } + } + return logs.String() +} + // GetVagrantLog returns the logs of on vagrant commands that initialize the nodes and provision K3s on each node. // It also attempts to fetch the systemctl logs of K3s on nodes where the k3s.service failed. func GetVagrantLog(cErr error) string { diff --git a/tests/e2e/token/token_test.go b/tests/e2e/token/token_test.go index bd0cc38a1fc8..3b3c011d6ae7 100644 --- a/tests/e2e/token/token_test.go +++ b/tests/e2e/token/token_test.go @@ -202,7 +202,9 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if !failed { + if failed { + AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } if !failed || *ci { diff --git a/tests/e2e/upgradecluster/upgradecluster_test.go b/tests/e2e/upgradecluster/upgradecluster_test.go index 18bd1cbee7b1..fab93a6bbd89 100644 --- a/tests/e2e/upgradecluster/upgradecluster_test.go +++ b/tests/e2e/upgradecluster/upgradecluster_test.go @@ -215,14 +215,13 @@ var _ = Describe("Verify Upgrade", Ordered, func() { }, "420s", "2s").Should(Succeed()) cmd := "kubectl --kubeconfig=" + kubeConfigFile + " exec volume-test -- sh -c 'echo local-path-test > /data/test'" - _, err = e2e.RunCommand(cmd) - Expect(err).NotTo(HaveOccurred()) + res, err := e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred(), "failed cmd: %q result: %s", cmd, res) fmt.Println("Data stored in pvc: local-path-test") cmd = "kubectl delete pod volume-test --kubeconfig=" + kubeConfigFile - res, err := e2e.RunCommand(cmd) - Expect(err).NotTo(HaveOccurred()) - fmt.Println(res) + res, err = e2e.RunCommand(cmd) + Expect(err).NotTo(HaveOccurred(), "failed cmd: %q result: %s", cmd, res) _, err = e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *hardened) Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed") @@ -245,7 +244,7 @@ var _ = Describe("Verify Upgrade", Ordered, func() { Eventually(func() (string, error) { cmd := "kubectl exec volume-test --kubeconfig=" + kubeConfigFile + " -- cat /data/test" return e2e.RunCommand(cmd) - }, "180s", "2s").Should(ContainSubstring("local-path-test")) + }, "180s", "2s").Should(ContainSubstring("local-path-test"), "Failed to retrieve data from pvc") }) It("Upgrades with no issues", func() { @@ -385,7 +384,9 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if !failed { + if failed { + AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } if !failed || *ci { diff --git a/tests/e2e/validatecluster/validatecluster_test.go b/tests/e2e/validatecluster/validatecluster_test.go index 8db46e673cfa..2c4807cce98d 100644 --- a/tests/e2e/validatecluster/validatecluster_test.go +++ b/tests/e2e/validatecluster/validatecluster_test.go @@ -381,7 +381,9 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if !failed { + if failed { + AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } if !failed || *ci { diff --git a/tests/e2e/wasm/wasm_test.go b/tests/e2e/wasm/wasm_test.go index 1e887a086a29..7fa216088b35 100644 --- a/tests/e2e/wasm/wasm_test.go +++ b/tests/e2e/wasm/wasm_test.go @@ -135,10 +135,12 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { - if failed && !*ci { - fmt.Println("FAILED!") + if failed { + AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) + } + if !failed || *ci { Expect(e2e.DestroyCluster()).To(Succeed()) Expect(os.Remove(kubeConfigFile)).To(Succeed()) }