diff --git a/inttest/Makefile.variables b/inttest/Makefile.variables index a25aa93bf59e..34e67f767ae1 100644 --- a/inttest/Makefile.variables +++ b/inttest/Makefile.variables @@ -56,6 +56,7 @@ smoketests := \ check-noderole-single \ check-openebs\ check-psp \ + check-reset \ check-singlenode \ check-statussocket \ check-upgrade \ diff --git a/inttest/backup/backup_test.go b/inttest/backup/backup_test.go index 8ed5c0fbcf3b..c522dd5c6acb 100644 --- a/inttest/backup/backup_test.go +++ b/inttest/backup/backup_test.go @@ -96,8 +96,8 @@ func (s *BackupSuite) TestK0sGetsUp() { s.Require().NoError(s.StopController(s.ControllerNode(0))) _ = s.StopController(s.ControllerNode(1)) // No error check as k0s might have actually exited since etcd is not really happy - s.Require().NoError(s.Reset(s.ControllerNode(0))) - s.Require().NoError(s.Reset(s.ControllerNode(1))) + s.reset(s.ControllerNode(0)) + s.reset(s.ControllerNode(1)) s.Require().NoError(s.restoreFunc()) s.Require().NoError(s.InitController(0, "--enable-worker")) @@ -122,6 +122,13 @@ func (s *BackupSuite) TestK0sGetsUp() { s.Require().NoError(s.VerifyFileSystemRestore()) } +func (s *BackupSuite) reset(name string) { + ssh, err := s.SSH(s.Context(), name) + s.Require().NoError(err) + defer ssh.Disconnect() + s.Require().NoError(ssh.Exec(s.Context(), "k0s reset --debug", common.SSHStreams{})) +} + type snapshot struct { namespaces map[types.UID]string services map[types.UID]string diff --git a/inttest/byocri/byocri_test.go b/inttest/byocri/byocri_test.go index 2c455065bae9..481162e684d7 100644 --- a/inttest/byocri/byocri_test.go +++ b/inttest/byocri/byocri_test.go @@ -18,6 +18,7 @@ package byocri import ( "fmt" + "strings" "testing" "github.com/avast/retry-go" @@ -31,21 +32,51 @@ type BYOCRISuite struct { common.BootlooseSuite } -func (s *BYOCRISuite) TestK0sGetsUp() { +func (s *BYOCRISuite) TestBringYourOwnCRI() { + ctx := s.Context() + workerNode := s.WorkerNode(0) - s.NoError(s.InitController(0)) - s.Require().NoError(s.runDockerWorker()) + if ok := s.Run("k0s gets up", func() { + s.NoError(s.InitController(0)) + s.Require().NoError(s.runDockerWorker()) - kc, err := s.KubeClient(s.ControllerNode(0)) - s.Require().NoError(err) + kc, err := s.KubeClient(s.ControllerNode(0)) + s.Require().NoError(err) - err = s.WaitForNodeReady(s.WorkerNode(0), kc) - s.NoError(err) + err = s.WaitForNodeReady(workerNode, kc) + s.NoError(err) - s.AssertSomeKubeSystemPods(kc) + s.AssertSomeKubeSystemPods(kc) - s.T().Log("waiting to see CNI pods ready") - s.NoError(common.WaitForKubeRouterReady(s.Context(), kc), "CNI did not start") + s.T().Log("waiting to see CNI pods ready") + s.NoError(common.WaitForKubeRouterReady(ctx, kc), "CNI did not start") + }); !ok { + return + } + + s.Run("k0s reset terminates Docker containers", func() { + ssh, err := s.SSH(ctx, workerNode) + s.Require().NoError(err) + defer ssh.Disconnect() + + var containerIDs strings.Builder + if s.NoError(ssh.Exec(ctx, "docker ps -q", common.SSHStreams{Out: &containerIDs}), "Failed to get running Docker containers") { + s.NotEmpty(containerIDs.String(), "Expected some running Docker containers") + } + + s.NoError(s.StopWorker(workerNode), "Failed to stop k0s") + + resetCmd := "k0s reset --debug --cri-socket remote:unix:///var/run/cri-dockerd.sock" + streams, flushStreams := common.TestLogStreams(s.T(), "reset") + err = ssh.Exec(ctx, resetCmd, streams) + flushStreams() + s.NoError(err, "k0s reset didn't exit cleanly") + + containerIDs.Reset() + if s.NoError(ssh.Exec(ctx, "docker ps -q", common.SSHStreams{Out: &containerIDs}), "Failed to get running Docker containers") { + s.Empty(containerIDs.String(), "Expected no running Docker containers") + } + }) } func (s *BYOCRISuite) runDockerWorker() error { diff --git a/inttest/common/bootloosesuite.go b/inttest/common/bootloosesuite.go index ecd9540452df..1f8585155130 100644 --- a/inttest/common/bootloosesuite.go +++ b/inttest/common/bootloosesuite.go @@ -795,15 +795,6 @@ func (s *BootlooseSuite) StopWorker(name string) error { return s.launchDelegate.StopWorker(s.Context(), ssh) } -func (s *BootlooseSuite) Reset(name string) error { - ssh, err := s.SSH(s.Context(), name) - s.Require().NoError(err) - defer ssh.Disconnect() - resetCommand := fmt.Sprintf("%s reset --debug", s.K0sFullPath) - _, err = ssh.ExecWithOutput(s.Context(), resetCommand) - return err -} - // KubeClient return kube client by loading the admin access config from given node func (s *BootlooseSuite) GetKubeConfig(node string, k0sKubeconfigArgs ...string) (*rest.Config, error) { machine, err := s.MachineForName(node) diff --git a/inttest/common/ssh.go b/inttest/common/ssh.go index 29e4889620fc..ef6ea3eee472 100644 --- a/inttest/common/ssh.go +++ b/inttest/common/ssh.go @@ -27,6 +27,7 @@ import ( "strconv" "strings" "sync" + "testing" "github.com/mitchellh/go-homedir" "golang.org/x/crypto/ssh" @@ -193,6 +194,16 @@ func (c *SSHConnection) Exec(ctx context.Context, cmd string, streams SSHStreams } } +// Returns SSH streams that log lines to the test log. +func TestLogStreams(t *testing.T, prefix string) (_ SSHStreams, flush func()) { + out := LineWriter{WriteLine: func(line []byte) { t.Logf("%s stdout: %s", prefix, string(line)) }} + err := LineWriter{WriteLine: func(line []byte) { t.Logf("%s stderr: %s", prefix, string(line)) }} + return SSHStreams{Out: &out, Err: &err}, func() { + out.Flush() + err.Flush() + } +} + func newWriterBuffer() (io.WriteCloser, func() []byte) { var mu sync.Mutex var buf bytes.Buffer diff --git a/inttest/common/util.go b/inttest/common/util.go index 39ff3a84a4bc..2e220cec7f5d 100644 --- a/inttest/common/util.go +++ b/inttest/common/util.go @@ -345,3 +345,44 @@ func logfFrom(ctx context.Context) LogfFn { } return logrus.Infof } + +type LineWriter struct { + WriteLine func([]byte) + buf []byte +} + +// Write implements [io.Writer]. +func (s *LineWriter) Write(in []byte) (int, error) { + s.buf = append(s.buf, in...) + s.logLines() + return len(in), nil +} + +// Logs each complete line and discards the used data. +func (s *LineWriter) logLines() { + var off int + for { + n := bytes.IndexByte(s.buf[off:], '\n') + if n < 0 { + break + } + + s.WriteLine(s.buf[off : off+n]) + off += n + 1 + } + + // Move the unprocessed data to the beginning of the buffer and reset the length. + if off > 0 { + len := copy(s.buf, s.buf[off:]) + s.buf = s.buf[:len] + } +} + +// Logs any remaining data in the buffer that doesn't end with a newline. +func (s *LineWriter) Flush() { + if len(s.buf) > 0 { + s.WriteLine(s.buf) + // Reset the length and keep the underlying array. + s.buf = s.buf[:0] + } +} diff --git a/inttest/reset/reset_test.go b/inttest/reset/reset_test.go new file mode 100644 index 000000000000..e34fcef8d989 --- /dev/null +++ b/inttest/reset/reset_test.go @@ -0,0 +1,82 @@ +/* +Copyright 2024 k0s authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reset + +import ( + "testing" + + testifysuite "github.com/stretchr/testify/suite" + + "github.com/k0sproject/k0s/inttest/common" +) + +type suite struct { + common.BootlooseSuite +} + +func (s *suite) TestReset() { + ctx := s.Context() + workerNode := s.WorkerNode(0) + + if ok := s.Run("k0s gets up", func() { + s.Require().NoError(s.InitController(0, "--disable-components=konnectivity-server,metrics-server")) + s.Require().NoError(s.RunWorkers()) + + kc, err := s.KubeClient(s.ControllerNode(0)) + s.Require().NoError(err) + + err = s.WaitForNodeReady(workerNode, kc) + s.NoError(err) + + s.T().Log("waiting to see CNI pods ready") + s.NoError(common.WaitForKubeRouterReady(ctx, kc), "CNI did not start") + }); !ok { + return + } + + s.Run("k0s reset", func() { + ssh, err := s.SSH(ctx, workerNode) + s.Require().NoError(err) + defer ssh.Disconnect() + + s.NoError(ssh.Exec(ctx, "test -d /var/lib/k0s", common.SSHStreams{}), "/var/lib/k0s is not a directory") + s.NoError(ssh.Exec(ctx, "test -d /run/k0s", common.SSHStreams{}), "/run/k0s is not a directory") + + s.NoError(ssh.Exec(ctx, "pidof containerd-shim-runc-v2 >&2", common.SSHStreams{}), "Expected some running containerd shims") + + s.NoError(s.StopWorker(workerNode), "Failed to stop k0s") + + streams, flushStreams := common.TestLogStreams(s.T(), "reset") + err = ssh.Exec(ctx, "k0s reset --debug", streams) + flushStreams() + s.NoError(err, "k0s reset didn't exit cleanly") + + // /var/lib/k0s is a mount point in the Docker container and can't be deleted, so it must be empty + s.NoError(ssh.Exec(ctx, `x="$(ls -A /var/lib/k0s)" && echo "$x" >&2 && [ -z "$x" ]`, common.SSHStreams{}), "/var/lib/k0s is not empty") + s.NoError(ssh.Exec(ctx, "! test -e /run/k0s", common.SSHStreams{}), "/run/k0s still exists") + s.NoError(ssh.Exec(ctx, "! pidof containerd-shim-runc-v2 >&2", common.SSHStreams{}), "Expected no running containerd shims") + }) +} + +func TestResetSuite(t *testing.T) { + testifysuite.Run(t, &suite{ + common.BootlooseSuite{ + ControllerCount: 1, + WorkerCount: 1, + }, + }) +}