diff --git a/test/pkg/integration/integration.go b/test/pkg/integration/integration.go index d37ad10219ffbc..f6a83b74ee8a32 100644 --- a/test/pkg/integration/integration.go +++ b/test/pkg/integration/integration.go @@ -187,20 +187,21 @@ func Instrument(component ComponentType, agentName string, namespace string, kub } execErrs := make(chan error, 1) - go func() { + execF := func() { defer close(execErrs) _, _, _, execErr := podExec.ExecCmd(cmd, podName, namespace, containerName) if execErr != nil { execErrs <- execErr } - }() + } + go execF() select { case err := <-execErrs: if err != nil { return nil, closer, err } return nil, closer, fmt.Errorf("agent stopped unexepectedly") - case <-time.After(1 * time.Second): + case <-time.After(30 * time.Second): } ctx, cancel := context.WithCancel(context.Background()) @@ -230,7 +231,7 @@ func Instrument(component ComponentType, agentName string, namespace string, kub var res *rpc.Client var lastError error - waitErr := wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) { + waitErr := wait.PollImmediate(5*time.Second, 3*time.Minute, func() (bool, error) { res, lastError = rpc.DialHTTP("tcp", fmt.Sprintf("localhost:%d", localAgentPort)) if lastError != nil { return false, nil diff --git a/test/pkg/integration/workspace.go b/test/pkg/integration/workspace.go index ad7c1e802b6c2b..e88129f17fd815 100644 --- a/test/pkg/integration/workspace.go +++ b/test/pkg/integration/workspace.go @@ -6,9 +6,7 @@ package integration import ( "context" - "fmt" "io" - "strings" "sync" "time" @@ -16,8 +14,6 @@ import ( "golang.org/x/xerrors" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" "github.com/gitpod-io/gitpod/common-go/namegen" csapi "github.com/gitpod-io/gitpod/content-service/api" @@ -28,7 +24,7 @@ import ( const ( gitpodBuiltinUserID = "builtin-user-workspace-probe-0000000" - perCallTimeout = 1 * time.Minute + perCallTimeout = 3 * time.Minute ) type launchWorkspaceDirectlyOptions struct { @@ -97,48 +93,53 @@ type LaunchWorkspaceDirectlyResult struct { // LaunchWorkspaceDirectly starts a workspace pod by talking directly to ws-manager. // Whenever possible prefer this function over LaunchWorkspaceFromContextURL, because // it has fewer prerequisites. -func LaunchWorkspaceDirectly(ctx context.Context, api *ComponentAPI, opts ...LaunchWorkspaceDirectlyOpt) (*LaunchWorkspaceDirectlyResult, error) { +func LaunchWorkspaceDirectly(ctx context.Context, api *ComponentAPI, opts ...LaunchWorkspaceDirectlyOpt) (*LaunchWorkspaceDirectlyResult, func(waitForStop bool) error, error) { options := launchWorkspaceDirectlyOptions{ BaseImage: "docker.io/gitpod/workspace-full:latest", } for _, o := range opts { err := o(&options) if err != nil { - return nil, err + return nil, nil, err } } instanceID, err := uuid.NewRandom() if err != nil { - return nil, err + return nil, nil, err } workspaceID, err := namegen.GenerateWorkspaceID() if err != nil { - return nil, err - + return nil, nil, err } var workspaceImage string if options.BaseImage != "" { - workspaceImage, err = resolveOrBuildImage(ctx, api, options.BaseImage) - if err != nil { - return nil, xerrors.Errorf("cannot resolve base image: %v", err) + for { + workspaceImage, err = resolveOrBuildImage(ctx, api, options.BaseImage) + if st, ok := status.FromError(err); ok && st.Code() == codes.Unavailable { + time.Sleep(5 * time.Second) + continue + } else if err != nil { + return nil, nil, xerrors.Errorf("cannot resolve base image: %v", err) + } + break } } if workspaceImage == "" { - return nil, xerrors.Errorf("cannot start workspaces without a workspace image (required by registry-facade resolver)") + return nil, nil, xerrors.Errorf("cannot start workspaces without a workspace image (required by registry-facade resolver)") } ideImage := options.IdeImage if ideImage == "" { cfg, err := GetServerIDEConfig(api.namespace, api.client) if err != nil { - return nil, xerrors.Errorf("cannot find server IDE config: %q", err) + return nil, nil, xerrors.Errorf("cannot find server IDE config: %q", err) } ideImage = cfg.IDEOptions.Options.Code.Image if ideImage == "" { - return nil, xerrors.Errorf("cannot start workspaces without an IDE image (required by registry-facade resolver)") + return nil, nil, xerrors.Errorf("cannot start workspaces without an IDE image (required by registry-facade resolver)") } } @@ -173,7 +174,7 @@ func LaunchWorkspaceDirectly(ctx context.Context, api *ComponentAPI, opts ...Lau for _, m := range options.Mods { err := m(req) if err != nil { - return nil, err + return nil, nil, err } } @@ -182,26 +183,56 @@ func LaunchWorkspaceDirectly(ctx context.Context, api *ComponentAPI, opts ...Lau wsm, err := api.WorkspaceManager() if err != nil { - return nil, xerrors.Errorf("cannot start workspace manager: %q", err) + return nil, nil, xerrors.Errorf("cannot start workspace manager: %q", err) } sresp, err := wsm.StartWorkspace(sctx, req) if err != nil { - return nil, xerrors.Errorf("cannot start workspace: %q", err) + return nil, nil, xerrors.Errorf("cannot start workspace: %q", err) + } + + stopWs := func(waitForStop bool) error { + tctx, tcancel := context.WithTimeout(context.Background(), perCallTimeout) + defer tcancel() + + for { + err = DeleteWorkspace(tctx, api, req.Id) + if st, ok := status.FromError(err); ok && st.Code() == codes.Unavailable { + time.Sleep(5 * time.Second) + continue + } else if err != nil { + return err + } + break + } + for { + _, err = WaitForWorkspaceStop(tctx, api, req.Id) + if st, ok := status.FromError(err); ok && st.Code() == codes.Unavailable { + time.Sleep(5 * time.Second) + continue + } else if err != nil { + return err + } + break + } + return err } + defer func() { + if err != nil { + stopWs(false) + } + }() lastStatus, err := WaitForWorkspaceStart(ctx, instanceID.String(), api, options.WaitForOpts...) if err != nil { - return nil, xerrors.Errorf("cannot wait for workspace start: %q", err) + return nil, nil, xerrors.Errorf("cannot wait for workspace start: %q", err) } - // it.t.Logf("workspace is running: instanceID=%s", instanceID.String()) - return &LaunchWorkspaceDirectlyResult{ Req: req, IdeURL: sresp.Url, LastStatus: lastStatus, - }, nil + }, stopWs, nil } // LaunchWorkspaceFromContextURL force-creates a new workspace using the Gitpod server API, @@ -246,9 +277,6 @@ func LaunchWorkspaceFromContextURL(ctx context.Context, contextURL string, usern sctx, scancel := context.WithTimeout(ctx, perCallTimeout) _ = server.StopWorkspace(sctx, resp.CreatedWorkspaceID) scancel() - //if err != nil { - //it.t.Errorf("cannot stop workspace: %q", err) - //} if waitForStop { _, _ = WaitForWorkspaceStop(ctx, api, nfo.LatestInstance.ID) @@ -413,7 +441,6 @@ func WaitForWorkspaceStop(ctx context.Context, api *ComponentAPI, instanceID str _ = sub.CloseSend() }() - var workspaceID string done := make(chan struct{}) errCh := make(chan error) go func() { @@ -432,7 +459,6 @@ func WaitForWorkspaceStop(ctx context.Context, api *ComponentAPI, instanceID str continue } - workspaceID = status.Metadata.MetaId if status.Conditions.Failed != "" { errCh <- xerrors.Errorf("workspace instance %s failed: %s", instanceID, status.Conditions.Failed) return @@ -441,6 +467,8 @@ func WaitForWorkspaceStop(ctx context.Context, api *ComponentAPI, instanceID str lastStatus = status return } + + time.Sleep(10 * time.Second) } }() diff --git a/test/tests/components/ws-manager/content_test.go b/test/tests/components/ws-manager/content_test.go index cc986f6ce9f7ca..d62ce50a5a5897 100644 --- a/test/tests/components/ws-manager/content_test.go +++ b/test/tests/components/ws-manager/content_test.go @@ -23,7 +23,7 @@ import ( func TestBackup(t *testing.T) { f := features.New("backup"). Assess("it should start a workspace, create a file and successfully create a backup", func(_ context.Context, t *testing.T, cfg *envconf.Config) context.Context { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() api := integration.NewComponentAPI(ctx, cfg.Namespace(), kubeconfig, cfg.Client()) @@ -31,22 +31,21 @@ func TestBackup(t *testing.T) { api.Done(t) }) - wsm, err := api.WorkspaceManager() - if err != nil { - t.Fatal(err) - } - - ws, err := integration.LaunchWorkspaceDirectly(ctx, api) + ws1, stopWs1, err := integration.LaunchWorkspaceDirectly(ctx, api) if err != nil { t.Fatal(err) } rsa, closer, err := integration.Instrument(integration.ComponentWorkspace, "workspace", cfg.Namespace(), kubeconfig, cfg.Client(), - integration.WithInstanceID(ws.Req.Id), + integration.WithInstanceID(ws1.Req.Id), integration.WithContainer("workspace"), integration.WithWorkspacekitLift(true), ) if err != nil { + err = stopWs1(true) + if err != nil { + t.Errorf("cannot stop workspace: %q", err) + } t.Fatal(err) } integration.DeferCloser(t, closer) @@ -57,53 +56,46 @@ func TestBackup(t *testing.T) { Content: []byte("hello world"), Mode: 0644, }, &resp) - if err != nil { - _, _ = wsm.StopWorkspace(ctx, &wsapi.StopWorkspaceRequest{Id: ws.Req.Id}) - t.Fatal(err) - } rsa.Close() - - _, err = wsm.StopWorkspace(ctx, &wsapi.StopWorkspaceRequest{ - Id: ws.Req.Id, - }) if err != nil { + err = stopWs1(true) + if err != nil { + t.Errorf("cannot stop workspace: %q", err) + } t.Fatal(err) } - _, err = integration.WaitForWorkspaceStop(ctx, api, ws.Req.Id) + err = stopWs1(true) if err != nil { t.Fatal(err) } - ws, err = integration.LaunchWorkspaceDirectly(ctx, api, + ws2, stopWs2, err := integration.LaunchWorkspaceDirectly(ctx, api, integration.WithRequestModifier(func(w *wsapi.StartWorkspaceRequest) error { - w.ServicePrefix = ws.Req.ServicePrefix - w.Metadata.MetaId = ws.Req.Metadata.MetaId - w.Metadata.Owner = ws.Req.Metadata.Owner + w.ServicePrefix = ws1.Req.ServicePrefix + w.Metadata.MetaId = ws1.Req.Metadata.MetaId + w.Metadata.Owner = ws1.Req.Metadata.Owner return nil }), ) if err != nil { t.Fatal(err) } + t.Cleanup(func() { + err = stopWs2(true) + if err != nil { + t.Errorf("cannot stop workspace: %q", err) + } + }) rsa, closer, err = integration.Instrument(integration.ComponentWorkspace, "workspace", cfg.Namespace(), kubeconfig, cfg.Client(), - integration.WithInstanceID(ws.Req.Id), + integration.WithInstanceID(ws2.Req.Id), ) if err != nil { t.Fatal(err) } integration.DeferCloser(t, closer) - defer func() { - t.Log("Cleaning up on TestBackup exit") - sctx, scancel := context.WithTimeout(ctx, 5*time.Second) - defer scancel() - _, _ = wsm.StopWorkspace(sctx, &wsapi.StopWorkspaceRequest{ - Id: ws.Req.Id, - }) - }() - var ls agent.ListDirResponse err = rsa.Call("WorkspaceAgent.ListDir", &agent.ListDirRequest{ Dir: "/workspace", @@ -146,13 +138,17 @@ func TestMissingBackup(t *testing.T) { api.Done(t) }) - ws, err := integration.LaunchWorkspaceDirectly(ctx, api) + ws, stopWs, err := integration.LaunchWorkspaceDirectly(ctx, api) if err != nil { t.Fatal(err) } wsm, err := api.WorkspaceManager() if err != nil { + err = stopWs(true) + if err != nil { + t.Errorf("cannot stop workspace: %q", err) + } t.Fatal(err) } @@ -189,7 +185,7 @@ func TestMissingBackup(t *testing.T) { } for _, test := range tests { t.Run(test.Name+"_backup_init", func(t *testing.T) { - testws, err := integration.LaunchWorkspaceDirectly(ctx, api, integration.WithRequestModifier(func(w *wsapi.StartWorkspaceRequest) error { + testws, stopWs, err := integration.LaunchWorkspaceDirectly(ctx, api, integration.WithRequestModifier(func(w *wsapi.StartWorkspaceRequest) error { w.ServicePrefix = ws.Req.ServicePrefix w.Metadata.MetaId = ws.Req.Metadata.MetaId w.Metadata.Owner = ws.Req.Metadata.Owner @@ -210,6 +206,10 @@ func TestMissingBackup(t *testing.T) { return } if testws.LastStatus.Conditions.Failed == "" { + err = stopWs(true) + if err != nil { + t.Errorf("cannot stop workspace: %q", err) + } t.Errorf("restarted workspace did not fail despite missing backup, %v", testws) } }) diff --git a/test/tests/components/ws-manager/prebuild_test.go b/test/tests/components/ws-manager/prebuild_test.go index a997e7c74ea0e0..f5b1a74550b1b3 100644 --- a/test/tests/components/ws-manager/prebuild_test.go +++ b/test/tests/components/ws-manager/prebuild_test.go @@ -28,7 +28,7 @@ func TestPrebuildWorkspaceTaskSuccess(t *testing.T) { api.Done(t) }) - ws, err := integration.LaunchWorkspaceDirectly(ctx, api, integration.WithRequestModifier(func(req *wsmanapi.StartWorkspaceRequest) error { + _, stopWs, err := integration.LaunchWorkspaceDirectly(ctx, api, integration.WithRequestModifier(func(req *wsmanapi.StartWorkspaceRequest) error { req.Type = wsmanapi.WorkspaceType_PREBUILD req.Spec.Envvars = append(req.Spec.Envvars, &wsmanapi.EnvironmentVariable{ Name: "GITPOD_TASKS", @@ -39,9 +39,11 @@ func TestPrebuildWorkspaceTaskSuccess(t *testing.T) { if err != nil { t.Fatalf("cannot launch a workspace: %q", err) } - t.Cleanup(func() { - _, _ = integration.WaitForWorkspaceStop(ctx, api, ws.Req.Id) + err = stopWs(true) + if err != nil { + t.Errorf("cannot stop workspace: %q", err) + } }) return ctx @@ -65,7 +67,7 @@ func TestPrebuildWorkspaceTaskFail(t *testing.T) { api.Done(t) }) - ws, err := integration.LaunchWorkspaceDirectly(ctx, api, integration.WithRequestModifier(func(req *wsmanapi.StartWorkspaceRequest) error { + ws, stopWs, err := integration.LaunchWorkspaceDirectly(ctx, api, integration.WithRequestModifier(func(req *wsmanapi.StartWorkspaceRequest) error { req.Type = wsmanapi.WorkspaceType_PREBUILD req.Spec.Envvars = append(req.Spec.Envvars, &wsmanapi.EnvironmentVariable{ Name: "GITPOD_TASKS", @@ -77,6 +79,13 @@ func TestPrebuildWorkspaceTaskFail(t *testing.T) { t.Fatalf("cannot start workspace: %q", err) } + t.Cleanup(func() { + err = stopWs(true) + if err != nil { + t.Errorf("cannot stop workspace: %q", err) + } + }) + _, err = integration.WaitForWorkspace(ctx, api, ws.Req.Id, func(status *wsmanapi.WorkspaceStatus) bool { if status.Phase != wsmanapi.WorkspacePhase_STOPPED { return false @@ -91,10 +100,6 @@ func TestPrebuildWorkspaceTaskFail(t *testing.T) { t.Fatalf("cannot start workspace: %q", err) } - t.Cleanup(func() { - _, _ = integration.WaitForWorkspaceStop(ctx, api, ws.Req.Id) - }) - return ctx }). Feature() diff --git a/test/tests/components/ws-manager/tasks_test.go b/test/tests/components/ws-manager/tasks_test.go index c2ec6e3861d199..5fc1dd938e8011 100644 --- a/test/tests/components/ws-manager/tasks_test.go +++ b/test/tests/components/ws-manager/tasks_test.go @@ -68,24 +68,24 @@ func TestRegularWorkspaceTasks(t *testing.T) { return nil } - nfo, err := integration.LaunchWorkspaceDirectly(ctx, api, integration.WithRequestModifier(addInitTask)) + nfo, stopWs, err := integration.LaunchWorkspaceDirectly(ctx, api, integration.WithRequestModifier(addInitTask)) if err != nil { t.Fatal(err) } - t.Cleanup(func() { - err = integration.DeleteWorkspace(ctx, api, nfo.Req.Id) - if err == nil { - _, _ = integration.WaitForWorkspaceStop(ctx, api, nfo.Req.Id) + defer func() { + err = stopWs(true) + if err != nil { + t.Errorf("cannot stop workspace: %q", err) } - }) + }() rsa, closer, err := integration.Instrument(integration.ComponentWorkspace, "workspace", cfg.Namespace(), kubeconfig, cfg.Client(), integration.WithInstanceID(nfo.Req.Id)) + integration.DeferCloser(t, closer) if err != nil { t.Fatalf("unexpected error instrumenting workspace: %v", err) } defer rsa.Close() - integration.DeferCloser(t, closer) var parsedResp struct { Result struct { diff --git a/test/tests/workspace/cgroup_v2_test.go b/test/tests/workspace/cgroup_v2_test.go index 5f7ad6357f8081..e9dfd6b181ac5f 100644 --- a/test/tests/workspace/cgroup_v2_test.go +++ b/test/tests/workspace/cgroup_v2_test.go @@ -34,16 +34,16 @@ func TestCgroupV2(t *testing.T) { api.Done(t) }) - ws, err := integration.LaunchWorkspaceDirectly(ctx, api) + ws, stopWs, err := integration.LaunchWorkspaceDirectly(ctx, api) if err != nil { t.Fatal(err) } - defer func() { - err = integration.DeleteWorkspace(ctx, api, ws.Req.Id) + t.Cleanup(func() { + err = stopWs(true) if err != nil { - t.Fatal(err) + t.Errorf("cannot stop workspace: %q", err) } - }() + }) rsa, closer, err := integration.Instrument(integration.ComponentWorkspace, "workspace", cfg.Namespace(), kubeconfig, cfg.Client(), integration.WithInstanceID(ws.Req.Id), integration.WithWorkspacekitLift(true)) if err != nil { diff --git a/test/tests/workspace/contexts_test.go b/test/tests/workspace/contexts_test.go index 70540f1ab42dc1..0de1313183320c 100644 --- a/test/tests/workspace/contexts_test.go +++ b/test/tests/workspace/contexts_test.go @@ -127,7 +127,7 @@ func runContextTests(t *testing.T, tests []ContextTest) { if err != nil { t.Fatal(err) } - defer stopWS(false) // we do not wait for stopped here as it does not matter for this test case and speeds things up + defer stopWS(true) rsa, closer, err := integration.Instrument(integration.ComponentWorkspace, "workspace", cfg.Namespace(), kubeconfig, cfg.Client(), integration.WithInstanceID(nfo.LatestInstance.ID)) if err != nil { diff --git a/test/tests/workspace/docker_test.go b/test/tests/workspace/docker_test.go index c36ceef713a88c..64f560247bfeb2 100644 --- a/test/tests/workspace/docker_test.go +++ b/test/tests/workspace/docker_test.go @@ -28,19 +28,15 @@ func TestRunDocker(t *testing.T) { api.Done(t) }) - ws, err := integration.LaunchWorkspaceDirectly(ctx, api) + ws, stopWs, err := integration.LaunchWorkspaceDirectly(ctx, api) if err != nil { t.Fatal(err) } t.Cleanup(func() { - sctx, scancel := context.WithTimeout(context.Background(), 5*time.Second) - defer scancel() - - err = integration.DeleteWorkspace(sctx, api, ws.Req.Id) + err = stopWs(true) if err != nil { - t.Fatal(err) + t.Errorf("cannot stop workspace: %q", err) } - _, _ = integration.WaitForWorkspaceStop(sctx, api, ws.Req.Id) }) rsa, closer, err := integration.Instrument(integration.ComponentWorkspace, "workspace", cfg.Namespace(), kubeconfig, cfg.Client(), integration.WithInstanceID(ws.Req.Id), integration.WithWorkspacekitLift(true)) diff --git a/test/tests/workspace/example_test.go b/test/tests/workspace/example_test.go index e34dcecb7bfd8b..e0d86c8bf8cb4e 100644 --- a/test/tests/workspace/example_test.go +++ b/test/tests/workspace/example_test.go @@ -81,15 +81,17 @@ func TestLaunchWorkspaceDirectly(t *testing.T) { api.Done(t) }) - nfo, err := integration.LaunchWorkspaceDirectly(ctx, api) + _, stopWs, err := integration.LaunchWorkspaceDirectly(ctx, api) if err != nil { t.Fatal(err) } - err = integration.DeleteWorkspace(ctx, api, nfo.Req.Id) - if err != nil { - t.Fatal(err) - } + t.Cleanup(func() { + err = stopWs(true) + if err != nil { + t.Errorf("cannot stop workspace: %q", err) + } + }) return ctx }). diff --git a/test/tests/workspace/k3s_test.go b/test/tests/workspace/k3s_test.go index 97b46e28fc23e4..ec7b3b2573c887 100644 --- a/test/tests/workspace/k3s_test.go +++ b/test/tests/workspace/k3s_test.go @@ -35,16 +35,16 @@ func TestK3s(t *testing.T) { api.Done(t) }) - ws, err := integration.LaunchWorkspaceDirectly(ctx, api) + ws, stopWs, err := integration.LaunchWorkspaceDirectly(ctx, api) if err != nil { t.Fatal(err) } - defer func() { - err = integration.DeleteWorkspace(ctx, api, ws.Req.Id) + t.Cleanup(func() { + err = stopWs(true) if err != nil { - t.Fatal(err) + t.Errorf("cannot stop workspace: %q", err) } - }() + }) rsa, closer, err := integration.Instrument(integration.ComponentWorkspace, "workspace", cfg.Namespace(), kubeconfig, cfg.Client(), integration.WithInstanceID(ws.Req.Id), integration.WithWorkspacekitLift(true)) if err != nil { diff --git a/test/tests/workspace/mount_proc_test.go b/test/tests/workspace/mount_proc_test.go index 7efdfba83d98ab..c4f811e6d2335d 100644 --- a/test/tests/workspace/mount_proc_test.go +++ b/test/tests/workspace/mount_proc_test.go @@ -56,11 +56,18 @@ func TestMountProc(t *testing.T) { api.Done(t) }) - ws, err := integration.LaunchWorkspaceDirectly(ctx, api) + ws, stopWs, err := integration.LaunchWorkspaceDirectly(ctx, api) if err != nil { t.Fatal(err) } + t.Cleanup(func() { + err = stopWs(true) + if err != nil { + t.Errorf("cannot stop workspace: %q", err) + } + }) + rsa, closer, err := integration.Instrument(integration.ComponentWorkspace, "workspace", cfg.Namespace(), kubeconfig, cfg.Client(), integration.WithInstanceID(ws.Req.Id), integration.WithWorkspacekitLift(true)) if err != nil { t.Fatalf("unexpected error instrumenting workspace: %v", err) @@ -78,11 +85,6 @@ func TestMountProc(t *testing.T) { } wg.Wait() - err = integration.DeleteWorkspace(ctx, api, ws.Req.Id) - if err != nil { - t.Fatal(err) - } - return ctx }). Feature()