From 69fdb89dc02b16daf1909a01a54495880045a156 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 15 Nov 2021 16:13:59 +0000 Subject: [PATCH 1/2] Setup/Add integration/deployment pytests --- .github/workflows/kubernetes_test.yaml | 6 +++- .github/workflows/test.yaml | 2 +- docs/source/dev_guide/testing.md | 11 +++++++ environment-dev.yaml | 4 +++ tests_deployment/__init__.py | 0 tests_deployment/constants.py | 5 +++ tests_deployment/test_dask_gateway.py | 32 ++++++++++++++++++ tests_deployment/utils.py | 45 ++++++++++++++++++++++++++ 8 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 tests_deployment/__init__.py create mode 100644 tests_deployment/constants.py create mode 100644 tests_deployment/test_dask_gateway.py create mode 100644 tests_deployment/utils.py diff --git a/.github/workflows/kubernetes_test.yaml b/.github/workflows/kubernetes_test.yaml index 5bea5e702d..a0e0161073 100644 --- a/.github/workflows/kubernetes_test.yaml +++ b/.github/workflows/kubernetes_test.yaml @@ -146,8 +146,12 @@ jobs: ./tests_e2e/cypress/screenshots/ ./tests_e2e/cypress/videos/ - ### CLEANUP AFTER CYPRESS + - name: Deployment Pytests + run: | + pip install dask-gateway kubernetes pytest dask-gateway + pytest tests_deployment/ -v + ### CLEANUP AFTER TESTS - name: Cleanup qhub deployment run: | cd local-deployment diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 59e72ee501..d8dde34fd3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -39,7 +39,7 @@ jobs: - name: Test QHub run: | pytest --version - pytest + pytest --ignore=tests_deployment test-render-providers: name: 'Test QHub Provider' diff --git a/docs/source/dev_guide/testing.md b/docs/source/dev_guide/testing.md index 746e5161dc..6ebf154dec 100644 --- a/docs/source/dev_guide/testing.md +++ b/docs/source/dev_guide/testing.md @@ -100,6 +100,17 @@ The final command above should open the Cypress UI where you can run the tests m Note that tests are heavily state dependent, so any changes or use of the deployed QHub could affect the results. +## Deployment/Integration Tests + +Deployment and Integration testing makes it easier to test various features of deployed QHub +on minikube such as Dask Gateway, external integrations, state of the kubernetes cluster via +simple Python code. You can run the integration and deployment tests via the following command: + +``` +pytest tests_deployment/ -v +``` + + # Cloud Testing Cloud testing on aws, gcp, azure, and digital ocean can be significantly more complicated and time consuming. But it is the only way to truly test the cloud deployments, including infrastructure, of course. To test on cloud Kubernetes, just deploy qhub in the normal way on those clouds, but using the [linked pip install](./index.md) of the qhub package. diff --git a/environment-dev.yaml b/environment-dev.yaml index 171eaadfef..53f856fae5 100644 --- a/environment-dev.yaml +++ b/environment-dev.yaml @@ -16,3 +16,7 @@ dependencies: - pytest - diagrams - jhub-client + + # deployment/integration tests dependencies + - dask-gateway + - python-kubernetes diff --git a/tests_deployment/__init__.py b/tests_deployment/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests_deployment/constants.py b/tests_deployment/constants.py new file mode 100644 index 0000000000..a37536eecd --- /dev/null +++ b/tests_deployment/constants.py @@ -0,0 +1,5 @@ +DASK_GATEWAY_JUPYTER_SECRET_NAME = "qhub-daskgateway-gateway" +JUPYTERHUB_TOKEN_SECRET_KEY_NAME = "jupyterhub_api_token" +NAMESPACE = 'dev' +QHUB_HOSTNAME = 'github-actions.qhub.dev' +GATEWAY_ENDPOINT = 'gateway' diff --git a/tests_deployment/test_dask_gateway.py b/tests_deployment/test_dask_gateway.py new file mode 100644 index 0000000000..039f10bfce --- /dev/null +++ b/tests_deployment/test_dask_gateway.py @@ -0,0 +1,32 @@ +import dask_gateway +import os + +import pytest +from tests_deployment import constants +from tests_deployment.utils import monkeypatch_ssl_context, get_jupyterhub_token + +monkeypatch_ssl_context() + + +@pytest.fixture +def dask_gateway_object(): + """Connects to Dask Gateway cluster from outside the cluster.""" + os.environ['JUPYTERHUB_API_TOKEN'] = get_jupyterhub_token() + return dask_gateway.Gateway( + address=f'https://{constants.QHUB_HOSTNAME}/{constants.GATEWAY_ENDPOINT}', + auth='jupyterhub', + proxy_address=f'tcp://{constants.QHUB_HOSTNAME}:8786' + ) + + +def test_dask_gateway(dask_gateway_object): + """This test checks if we're able to connect to dask gateway.""" + assert dask_gateway_object.list_clusters() == [] + + +def test_dask_gateway_cluster_options(dask_gateway_object): + """Tests Dask Gateway's cluster options.""" + cluster_options = dask_gateway_object.cluster_options() + assert cluster_options.conda_environment == "dask" + assert cluster_options.profile == "Small Worker" + assert cluster_options.environment_vars == {} diff --git a/tests_deployment/utils.py b/tests_deployment/utils.py new file mode 100644 index 0000000000..ba291c36c5 --- /dev/null +++ b/tests_deployment/utils.py @@ -0,0 +1,45 @@ +import base64 +import ssl + +from kubernetes import client, config + +from tests_deployment import constants + + +def get_kubernetes_api_instance(): + """Returns the v1 core Kubernetes api instance for making + calls to the kubernetes cluster + """ + config.load_kube_config() + return client.CoreV1Api() + + +def get_jupyterhub_token(): + """ + It fetches the secret that has the JupyterHub token to be able to + connect to dask gateway. + """ + v1 = get_kubernetes_api_instance() + secret = str(v1.read_namespaced_secret( + constants.DASK_GATEWAY_JUPYTER_SECRET_NAME, constants.NAMESPACE + ).data) + base64_encoded_token = eval(secret)[constants.JUPYTERHUB_TOKEN_SECRET_KEY_NAME] + return base64.b64decode(base64_encoded_token).decode() + + +def monkeypatch_ssl_context(): + """ + This is a workaround monkeypatch to disable ssl checking to avoid SSL + failures. + TODO: A better way to do this would be adding the Traefik's default certificate's + CA public key to the trusted certificate authorities. + """ + def create_default_context(context): + def _inner(*args, **kwargs): + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + return context + return _inner + + sslcontext = ssl.create_default_context() + ssl.create_default_context = create_default_context(sslcontext) From fa7114253596b68e17b0d612b853184531972d63 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Tue, 16 Nov 2021 11:48:45 +0000 Subject: [PATCH 2/2] Move dev dependencies to setup.py --- .github/workflows/kubernetes_test.yaml | 1 - setup.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/kubernetes_test.yaml b/.github/workflows/kubernetes_test.yaml index a0e0161073..40f720d1bc 100644 --- a/.github/workflows/kubernetes_test.yaml +++ b/.github/workflows/kubernetes_test.yaml @@ -148,7 +148,6 @@ jobs: - name: Deployment Pytests run: | - pip install dask-gateway kubernetes pytest dask-gateway pytest tests_deployment/ -v ### CLEANUP AFTER TESTS diff --git a/setup.py b/setup.py index c8b7487d88..b185247927 100644 --- a/setup.py +++ b/setup.py @@ -53,6 +53,8 @@ "diagrams", "jhub-client", "pre-commit", + "kubernetes", + "dask-gateway", ], }, include_package_data=True,