[tests] fix integration tests

Author: sagor999

test/README.md

@@ -40,33 +40,32 @@ You may want to run tests to assert whether a Gitpod installation is successfull
 
 Best for when you want to validate an environment.
 
-1. Create a service account, role, and role-binding for integration testing.
+1. Apply yaml file that will add all necessary permissions and create a job that will run tests.
    * [`kubectl apply -f ./integration.yaml`](./integration.yaml)
-2. Run a pod to execute the tests like so:
-    ```bash
-    # Replace <number>, <namespace>, and <gitpod_username> with meaningful values
-    kubectl run --image=eu.gcr.io/gitpod-core-dev/build/integration-tests:main.<number> \
-    integ-tests --serviceaccount=integration-svc \
-    --restart=Never \
-    --requests='cpu=2,memory=4Gi' \
-    -- /bin/sh -namespace=<namespace> -username=<gitpod_username>
-    ```
-3. Check logs to inspect test results like so `kubectl logs -f integ-tests`.
-4. Tear down the integration user and pod when testing is done.
+2. Check logs to inspect test results like so `kubectl logs -f jobs/integration-job`.
+3. Tear down the integration user and job when testing is done.
    * [`kubectl delete -f ./integration.yaml`](./integration.yaml)
-   * `kubectl delete pod integ-user`
 
 #### Go test
 
 Best for when you're actively developing Gitpod.
-
-1. Set your kubectl context to the cluster you want to test
-2. Integrate the Gitpod installation with OAuth for Github and/or Gitlab, otherwise related tests may fail
-3. Clone this repo, and `cd` to `./gitpod/test`
-4. Run the tests like so
+Test will work if images that they use are already cached by gitpod instnance. If not, they might fail if it takes too long to pull an image.
+There are 4 different types of tests:
+1. Enterprise specific, that require valid license to be installed. Run those with `-enterprise=true`
+2. Tests that require correct user (user should have github OAuth integration setup with gitpod). Run those with `-username=<gitpod_username>`. Make sure to load https://github.com/gitpod-io/gitpod-test-repo and https://github.com/gitpod-io/gitpod in gitpod env that you are testing to verify that it is able to pull an image. Wait for it to finish pulling gitpod image, this will ensure that test will not fail due to timeout while waiting to pull an image for the first time.
+3. To test gitlab integration, add `-gitlab=true`
+4. All other tests.
+
+To run the tests:
+1. Clone this repo, and `cd` to `./gitpod/test`
+2. Run the tests like so
   ```bash
   go test -v ./... \
   -kubeconfig=<path_to_kube_config_file> \
   -namespace=<namespace_where_gitpod_is_installed> \
-  -username=<a_user_in_the_gitpod_database>
+  -username=<gitpod_user_with_oauth_setup> \
+  -enterprise=<true|false> \
+  -gitlab=<true|false>
   ```
+3. Tests `TestUploadDownloadBlob` and `TestUploadDownloadBlobViaServer` will fail when testing locally, as they are trying to connect to cluster local resources directly. To test them use docker image instead that runs within the cluster.
+4. If you want to run specific test, add `-run <test>` before `-kubeconfig` parameter.

test/integration.yaml

@@ -15,16 +15,17 @@ rules:
         - 'secrets'
         - 'services'
         - 'configmaps'
+        - 'endpoints'
       verbs:
         - 'list'
         - 'get'
     - apiGroups:
         - ''
       resources:
         - 'pods/portforward'
+        - 'pods/exec'
       verbs:
         - 'create'
-
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
@@ -37,3 +38,23 @@ roleRef:
     kind: Role
     name: integration-role
     apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: integration-job
+spec:
+  template:
+    spec:
+      serviceAccountName: integration-svc
+      containers:
+      - name: tests
+        image: eu.gcr.io/gitpod-core-dev/build/integration-tests:kyleb-installer-integration.24
+        imagePullPolicy: Always
+        args: ["-username=sagor999"]
+        resources:
+          requests:
+            cpu: 2
+            memory: 4Gi
+      restartPolicy: Never
+  backoffLimit: 0

test/pkg/integration/apis.go

@@ -46,10 +46,11 @@ import (
 )
 
 // API provides access to the individual component's API
-func NewComponentAPI(ctx context.Context, namespace string, client klient.Client) *ComponentAPI {
+func NewComponentAPI(ctx context.Context, namespace string, kubeconfig string, client klient.Client) *ComponentAPI {
 	return &ComponentAPI{
-		namespace: namespace,
-		client:    client,
+		namespace:  namespace,
+		kubeconfig: kubeconfig,
+		client:     client,
 
 		closerMutex: sync.Mutex{},
 
@@ -71,8 +72,9 @@ type serverStatus struct {
 
 // ComponentAPI provides access to the individual component's API
 type ComponentAPI struct {
-	namespace string
-	client    klient.Client
+	namespace  string
+	kubeconfig string
+	client     klient.Client
 
 	closer      []func() error
 	closerMutex sync.Mutex
@@ -170,7 +172,7 @@ func (c *ComponentAPI) Supervisor(instanceID string) (grpc.ClientConnInterface,
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
-	ready, errc := common.ForwardPort(ctx, c.client.RESTConfig(), c.namespace, pod, fmt.Sprintf("%d:22999", localPort))
+	ready, errc := common.ForwardPort(ctx, c.kubeconfig, c.namespace, pod, fmt.Sprintf("%d:22999", localPort))
 	select {
 	case err = <-errc:
 		cancel()
@@ -483,7 +485,7 @@ func (c *ComponentAPI) WorkspaceManager() (wsmanapi.WorkspaceManagerClient, erro
 		}
 
 		ctx, cancel := context.WithCancel(context.Background())
-		ready, errc := common.ForwardPort(ctx, c.client.RESTConfig(), c.namespace, pod, fmt.Sprintf("%d:8080", localPort))
+		ready, errc := common.ForwardPort(ctx, c.kubeconfig, c.namespace, pod, fmt.Sprintf("%d:8080", localPort))
 		select {
 		case err := <-errc:
 			cancel()
@@ -556,7 +558,7 @@ func (c *ComponentAPI) BlobService() (csapi.BlobServiceClient, error) {
 		}
 
 		ctx, cancel := context.WithCancel(context.Background())
-		ready, errc := common.ForwardPort(ctx, c.client.RESTConfig(), c.namespace, pod, fmt.Sprintf("%d:8080", localPort))
+		ready, errc := common.ForwardPort(ctx, c.kubeconfig, c.namespace, pod, fmt.Sprintf("%d:8080", localPort))
 		select {
 		case err := <-errc:
 			cancel()
@@ -609,7 +611,7 @@ func (c *ComponentAPI) DB(options ...DBOpt) (*sql.DB, error) {
 	// if configured: setup local port-forward to DB pod
 	if config.ForwardPort != nil {
 		ctx, cancel := context.WithCancel(context.Background())
-		ready, errc := common.ForwardPort(ctx, c.client.RESTConfig(), c.namespace, config.ForwardPort.PodName, fmt.Sprintf("%d:%d", config.Port, config.ForwardPort.RemotePort))
+		ready, errc := common.ForwardPort(ctx, c.kubeconfig, c.namespace, config.ForwardPort.PodName, fmt.Sprintf("%d:%d", config.Port, config.ForwardPort.RemotePort))
 		select {
 		case err := <-errc:
 			cancel()
@@ -855,7 +857,7 @@ func (c *ComponentAPI) ImageBuilder(opts ...APIImageBuilderOpt) (imgbldr.ImageBu
 			}
 
 			ctx, cancel := context.WithCancel(context.Background())
-			ready, errc := common.ForwardPort(ctx, c.client.RESTConfig(), c.namespace, pod, fmt.Sprintf("%d:8080", localPort))
+			ready, errc := common.ForwardPort(ctx, c.kubeconfig, c.namespace, pod, fmt.Sprintf("%d:8080", localPort))
 			select {
 			case err = <-errc:
 				cancel()
@@ -904,7 +906,7 @@ func (c *ComponentAPI) ContentService() (ContentService, error) {
 		}
 
 		ctx, cancel := context.WithCancel(context.Background())
-		ready, errc := common.ForwardPort(ctx, c.client.RESTConfig(), c.namespace, pod, fmt.Sprintf("%d:8080", localPort))
+		ready, errc := common.ForwardPort(ctx, c.kubeconfig, c.namespace, pod, fmt.Sprintf("%d:8080", localPort))
 		select {
 		case err := <-errc:
 			cancel()

test/pkg/integration/common/port_forward.go

@@ -13,12 +13,12 @@ import (
 	"time"
 
 	"golang.org/x/xerrors"
-	"k8s.io/client-go/rest"
 )
 
 // ForwardPort establishes a TCP port forwarding to a Kubernetes pod
 // Uses kubectl instead of Go to use a local process that can reproduce the same behavior outside the tests
-func ForwardPort(ctx context.Context, config *rest.Config, namespace, pod, port string) (readychan chan struct{}, errchan chan error) {
+// Since we are using kubectl directly we need to pass kubeconfig explicetly
+func ForwardPort(ctx context.Context, kubeconfig string, namespace, pod, port string) (readychan chan struct{}, errchan chan error) {
 	errchan = make(chan error, 1)
 	readychan = make(chan struct{}, 1)
 
@@ -28,18 +28,19 @@ func ForwardPort(ctx context.Context, config *rest.Config, namespace, pod, port
 			"--address=0.0.0.0",
 			fmt.Sprintf("pod/%v", pod),
 			fmt.Sprintf("--namespace=%v", namespace),
+			fmt.Sprintf("--kubeconfig=%v", kubeconfig),
 			port,
 		}
 
 		command := exec.CommandContext(ctx, "kubectl", args...)
 		err := command.Start()
 		if err != nil {
-			errchan <- xerrors.Errorf("unexpected error starting port-forward: %w", err)
+			errchan <- xerrors.Errorf("unexpected error starting port-forward: %w, args: %v", err, args)
 		}
 
 		err = command.Wait()
 		if err != nil {
-			errchan <- xerrors.Errorf("unexpected error running port-forward: %w", err)
+			errchan <- xerrors.Errorf("unexpected error running port-forward: %w, args: %v", err, args)
 		}
 	}()
 

test/pkg/integration/integration.go

@@ -135,7 +135,7 @@ func WithWorkspacekitLift(lift bool) InstrumentOption {
 // If there isn't, we attempt to build `<agentName>_agent/main.go`.
 // The binary is copied to the destination pod, started and port-forwarded. Then we
 // create an RPC client.
-func Instrument(component ComponentType, agentName string, namespace string, client klient.Client, opts ...InstrumentOption) (*rpc.Client, []func() error, error) {
+func Instrument(component ComponentType, agentName string, namespace string, kubeconfig string, client klient.Client, opts ...InstrumentOption) (*rpc.Client, []func() error, error) {
 	var closer []func() error
 
 	options := instrumentOptions{
@@ -188,7 +188,7 @@ func Instrument(component ComponentType, agentName string, namespace string, cli
 	execErrs := make(chan error, 1)
 	go func() {
 		defer close(execErrs)
-		execErr := executeAgent(cmd, podName, containerName, namespace, client)
+		execErr := executeAgent(cmd, podName, containerName, namespace, kubeconfig, client)
 		if execErr != nil {
 			execErrs <- execErr
 		}
@@ -214,7 +214,7 @@ func Instrument(component ComponentType, agentName string, namespace string, cli
 		}
 	}()
 
-	fwdReady, fwdErr := common.ForwardPort(ctx, client.RESTConfig(), namespace, podName, strconv.Itoa(localAgentPort))
+	fwdReady, fwdErr := common.ForwardPort(ctx, kubeconfig, namespace, podName, strconv.Itoa(localAgentPort))
 	select {
 	case <-fwdReady:
 	case err := <-execErrs:
@@ -274,8 +274,9 @@ func getFreePort() (int, error) {
 	return result.Port, nil
 }
 
-func executeAgent(cmd []string, pod, container string, namespace string, client klient.Client) error {
-	args := []string{"exec", pod, fmt.Sprintf("--namespace=%v", namespace)}
+func executeAgent(cmd []string, pod, container string, namespace string, kubeconfig string, client klient.Client) error {
+	// since we are self signing certs for gitpod components, pass insecure flag
+	args := []string{"exec", pod, fmt.Sprintf("--namespace=%v", namespace), "--insecure-skip-tls-verify=true", fmt.Sprintf("--kubeconfig=%v", kubeconfig)}
 	if len(container) > 0 {
 		args = append(args, fmt.Sprintf("--container=%s", container))
 	}
@@ -285,7 +286,7 @@ func executeAgent(cmd []string, pod, container string, namespace string, client
 	command := exec.Command("kubectl", args...)
 	out, err := command.CombinedOutput()
 	if err != nil {
-		return xerrors.Errorf("cannot run kubectl command: %w\n%v", err, string(out))
+		return xerrors.Errorf("cannot run kubectl command: %w\n%v\nargs:%v", err, string(out), args)
 	}
 
 	return nil
@@ -431,12 +432,16 @@ func GetServerConfig(namespace string, client klient.Client) (*ServerConfigParti
 // ServerIDEConfigPartial is the subset of server IDE config we're using for integration tests.
 // NOTE: keep in sync with chart/templates/server-ide-configmap.yaml
 type ServerIDEConfigPartial struct {
-	IDEVersion      string `json:"ideVersion"`
-	IDEImageRepo    string `json:"ideImageRepo"`
-	IDEImageAliases struct {
-		Code       string `json:"code"`
-		CodeLatest string `json:"code-latest"`
-	} `json:"ideImageAliases"`
+	IDEOptions struct {
+		Options struct {
+			Code struct {
+				Image string `json:"image"`
+			} `json:"code"`
+			CodeLatest struct {
+				Image string `json:"image"`
+			} `json:"code-latest"`
+		} `json:"options"`
+	} `json:"ideOptions"`
 }
 
 func GetServerIDEConfig(namespace string, client klient.Client) (*ServerIDEConfigPartial, error) {

test/pkg/integration/setup.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"math/rand"
 	"os"
 	"testing"
 	"time"
@@ -29,9 +30,40 @@ func SkipWithoutUsername(t *testing.T, username string) {
 		t.Skip("Skipping because requires a username")
 	}
 }
-func Setup(ctx context.Context) (string, string, env.Environment) {
+
+func SkipWithoutEnterpriseLicense(t *testing.T, enterpise bool) {
+	if !enterpise {
+		t.Skip("Skipping because requires enterprise license")
+	}
+}
+
+func EnsureUserExists(t *testing.T, username string, api *ComponentAPI) string {
+	if username == "" {
+		t.Logf("no username provided, creating temporary one")
+		rand.Seed(time.Now().UnixNano())
+		randN := rand.Intn(1000)
+		new_user := fmt.Sprintf("johndoe%d", randN)
+		userId, err := CreateUser(new_user, false, api)
+		if err != nil {
+			t.Fatalf("cannot create user: %q", err)
+		}
+		t.Cleanup(func() {
+			err := DeleteUser(userId, api)
+			if err != nil {
+				t.Fatalf("error deleting user %q", err)
+			}
+		})
+		t.Logf("user '%s' with ID %s created", new_user, userId)
+		return new_user
+	}
+	return username
+}
+
+func Setup(ctx context.Context) (string, string, env.Environment, bool, string, bool) {
 	var (
 		username        string
+		enterprise      bool
+		gitlab          bool
 		waitGitpodReady time.Duration
 
 		namespace  string
@@ -46,6 +78,8 @@ func Setup(ctx context.Context) (string, string, env.Environment) {
 	klog.InitFlags(flagset)
 
 	flagset.StringVar(&username, "username", "", "username to execute the tests with. Chooses one automatically if left blank.")
+	flagset.BoolVar(&enterprise, "enterprise", false, "whether to test enterprise features. requires enterprise lisence installed.")
+	flagset.BoolVar(&gitlab, "gitlab", false, "whether to test gitlab integration.")
 	flagset.DurationVar(&waitGitpodReady, "wait-gitpod-timeout", 5*time.Minute, `wait time for Gitpod components before starting integration test`)
 	flagset.StringVar(&namespace, "namespace", "", "Kubernetes cluster namespaces to use")
 	flagset.StringVar(&kubeconfig, "kubeconfig", "", "The path to the kubeconfig file")
@@ -90,7 +124,7 @@ func Setup(ctx context.Context) (string, string, env.Environment) {
 		waitOnGitpodRunning(e.Namespace(), waitGitpodReady),
 	)
 
-	return username, e.Namespace(), testenv
+	return username, e.Namespace(), testenv, enterprise, kubeconfig, gitlab
 }
 
 func waitOnGitpodRunning(namespace string, waitTimeout time.Duration) env.Func {

test/pkg/integration/workspace.go

@@ -136,7 +136,7 @@ func LaunchWorkspaceDirectly(ctx context.Context, api *ComponentAPI, opts ...Lau
 		if err != nil {
 			return nil, xerrors.Errorf("cannot find server IDE config: %q", err)
 		}
-		ideImage = cfg.IDEImageAliases.Code
+		ideImage = cfg.IDEOptions.Options.Code.Image
 		if ideImage == "" {
 			return nil, xerrors.Errorf("cannot start workspaces without an IDE image (required by registry-facade resolver)")
 		}