nebari cli environment variable handling, support, keycloak, dev tests

src/_nebari/keycloak.py

@@ -114,7 +114,9 @@ def get_keycloak_admin_from_config(config: schema.Main):
 
 def keycloak_rest_api_call(config: schema.Main = None, request: str = None):
     """Communicate directly with the Keycloak REST API by passing it a request"""
-    keycloak_server_url = f"https://{config.domain}/auth/"
+    keycloak_server_url = os.environ.get(
+        "KEYCLOAK_SERVER_URL", f"https://{config.domain}/auth/"
+    )
 
     keycloak_admin_username = os.environ.get("KEYCLOAK_ADMIN_USERNAME", "root")
     keycloak_admin_password = os.environ.get(

src/_nebari/stages/infrastructure/__init__.py

@@ -315,7 +315,7 @@ class GCPNodeGroup(schema.Base):
 
 
 class GoogleCloudPlatformProvider(schema.Base):
-    project: str = pydantic.Field(default_factory=lambda: os.environ["PROJECT_ID"])
+    project: str = pydantic.Field(default_factory=lambda: os.environ.get("PROJECT_ID"))
     region: str = "us-central1"
     availability_zones: typing.Optional[typing.List[str]] = []
     kubernetes_version: typing.Optional[str]

src/_nebari/subcommands/dev.py

@@ -50,6 +50,6 @@ def keycloak_api(
 
         config_schema = nebari_plugin_manager.config_schema
 
-        read_configuration(config_filename, config_schema=config_schema)
-        r = keycloak_rest_api_call(config_filename, request=request)
+        config = read_configuration(config_filename, config_schema=config_schema)
+        r = keycloak_rest_api_call(config, request=request)
         print(json.dumps(r, indent=4))

src/_nebari/subcommands/support.py

@@ -2,6 +2,7 @@
 from zipfile import ZipFile
 
 import kubernetes.client
+import kubernetes.client.exceptions
 import kubernetes.config
 import typer
 
@@ -34,13 +35,13 @@ def support(
         """
         from nebari.plugins import nebari_plugin_manager
 
+        config_schema = nebari_plugin_manager.config_schema
+        namespace = read_configuration(config_filename, config_schema).namespace
+
         kubernetes.config.kube_config.load_kube_config()
 
         v1 = kubernetes.client.CoreV1Api()
 
-        config_schema = nebari_plugin_manager.config_schema
-        namespace = read_configuration(config_filename, config_schema).namespace
-
         pods = v1.list_namespaced_pod(namespace=namespace)
 
         for pod in pods.items:
@@ -72,9 +73,10 @@ def support(
                                 namespace=namespace,
                                 container=container,
                             )
+                            + "\n"
                         )
 
-                except client.exceptions.ApiException as e:
+                except kubernetes.client.exceptions.ApiException as e:
                     file.write("%s not available" % pod.metadata.name)
                     raise e
 

tests/tests_unit/test_cli_dev.py

@@ -0,0 +1,253 @@
+import json
+import tempfile
+from pathlib import Path
+from typing import Any, List
+from unittest.mock import Mock, patch
+
+import pytest
+import requests.exceptions
+import yaml
+from typer.testing import CliRunner
+
+from _nebari.cli import create_cli
+
+TEST_KEYCLOAKAPI_REQUEST = "GET /"  # get list of realms
+
+TEST_DOMAIN = "nebari.example.com"
+MOCK_KEYCLOAK_ENV = {
+    "KEYCLOAK_SERVER_URL": f"https://{TEST_DOMAIN}/auth/",
+    "KEYCLOAK_ADMIN_USERNAME": "root",
+    "KEYCLOAK_ADMIN_PASSWORD": "super-secret-123!",
+}
+
+TEST_ACCESS_TOKEN = "abc123"
+
+TEST_REALMS = [
+    {"id": "test-realm", "realm": "test-realm"},
+    {"id": "master", "realm": "master"},
+]
+
+runner = CliRunner()
+
+
+@pytest.mark.parametrize(
+    "args, exit_code, content",
+    [
+        # --help
+        ([], 0, ["Usage:"]),
+        (["--help"], 0, ["Usage:"]),
+        (["-h"], 0, ["Usage:"]),
+        (["keycloak-api", "--help"], 0, ["Usage:"]),
+        (["keycloak-api", "-h"], 0, ["Usage:"]),
+        # error, missing args
+        (["keycloak-api"], 2, ["Missing option"]),
+        (["keycloak-api", "--config"], 2, ["requires an argument"]),
+        (["keycloak-api", "-c"], 2, ["requires an argument"]),
+        (["keycloak-api", "--request"], 2, ["requires an argument"]),
+        (["keycloak-api", "-r"], 2, ["requires an argument"]),
+    ],
+)
+def test_cli_dev_stdout(args: List[str], exit_code: int, content: List[str]):
+    app = create_cli()
+    result = runner.invoke(app, ["dev"] + args)
+    assert result.exit_code == exit_code
+    for c in content:
+        assert c in result.stdout
+
+
+def mock_api_post(admin_password: str, url: str, headers: Any, data: Any, verify: bool):
+    response = Mock()
+    if (
+        url
+        == f"{MOCK_KEYCLOAK_ENV['KEYCLOAK_SERVER_URL']}realms/master/protocol/openid-connect/token"
+        and data["password"] == admin_password
+    ):
+        response.status_code = 200
+        response.content = bytes(
+            json.dumps({"access_token": TEST_ACCESS_TOKEN}), "UTF-8"
+        )
+    else:
+        response.status_code = 403
+    return response
+
+
+def mock_api_request(
+    access_token: str, method: str, url: str, headers: Any, verify: bool
+):
+    response = Mock()
+    if (
+        method == "GET"
+        and url == f"{MOCK_KEYCLOAK_ENV['KEYCLOAK_SERVER_URL']}admin/realms/"
+        and headers["Authorization"] == f"Bearer {access_token}"
+    ):
+        response.status_code = 200
+        response.content = bytes(json.dumps(TEST_REALMS), "UTF-8")
+    else:
+        response.status_code = 403
+    return response
+
+
+@patch(
+    "_nebari.keycloak.requests.post",
+    side_effect=lambda url, headers, data, verify: mock_api_post(
+        MOCK_KEYCLOAK_ENV["KEYCLOAK_ADMIN_PASSWORD"], url, headers, data, verify
+    ),
+)
+@patch(
+    "_nebari.keycloak.requests.request",
+    side_effect=lambda method, url, headers, verify: mock_api_request(
+        TEST_ACCESS_TOKEN, method, url, headers, verify
+    ),
+)
+def test_cli_dev_keycloakapi_happy_path_from_env(
+    _mock_requests_post, _mock_requests_request
+):
+    result = run_cli_dev(use_env=True)
+
+    assert 0 == result.exit_code
+    assert not result.exception
+
+    r = json.loads(result.stdout)
+    assert 2 == len(r)
+    assert "test-realm" == r[0]["realm"]
+
+
+@patch(
+    "_nebari.keycloak.requests.post",
+    side_effect=lambda url, headers, data, verify: mock_api_post(
+        MOCK_KEYCLOAK_ENV["KEYCLOAK_ADMIN_PASSWORD"], url, headers, data, verify
+    ),
+)
+@patch(
+    "_nebari.keycloak.requests.request",
+    side_effect=lambda method, url, headers, verify: mock_api_request(
+        TEST_ACCESS_TOKEN, method, url, headers, verify
+    ),
+)
+def test_cli_dev_keycloakapi_happy_path_from_config(
+    _mock_requests_post, _mock_requests_request
+):
+    result = run_cli_dev(use_env=False)
+
+    assert 0 == result.exit_code
+    assert not result.exception
+
+    r = json.loads(result.stdout)
+    assert 2 == len(r)
+    assert "test-realm" == r[0]["realm"]
+
+
+@patch(
+    "_nebari.keycloak.requests.post",
+    side_effect=lambda url, headers, data, verify: mock_api_post(
+        MOCK_KEYCLOAK_ENV["KEYCLOAK_ADMIN_PASSWORD"], url, headers, data, verify
+    ),
+)
+def test_cli_dev_keycloakapi_error_bad_request(_mock_requests_post):
+    result = run_cli_dev(request="malformed")
+
+    assert 1 == result.exit_code
+    assert result.exception
+    assert "not enough values to unpack" in str(result.exception)
+
+
+@patch(
+    "_nebari.keycloak.requests.post",
+    side_effect=lambda url, headers, data, verify: mock_api_post(
+        "invalid_admin_password", url, headers, data, verify
+    ),
+)
+def test_cli_dev_keycloakapi_error_authentication(_mock_requests_post):
+    result = run_cli_dev()
+
+    assert 1 == result.exit_code
+    assert result.exception
+    assert "Unable to retrieve Keycloak API token" in str(result.exception)
+    assert "Status code: 403" in str(result.exception)
+
+
+@patch(
+    "_nebari.keycloak.requests.post",
+    side_effect=lambda url, headers, data, verify: mock_api_post(
+        MOCK_KEYCLOAK_ENV["KEYCLOAK_ADMIN_PASSWORD"], url, headers, data, verify
+    ),
+)
+@patch(
+    "_nebari.keycloak.requests.request",
+    side_effect=lambda method, url, headers, verify: mock_api_request(
+        "invalid_access_token", method, url, headers, verify
+    ),
+)
+def test_cli_dev_keycloakapi_error_authorization(
+    _mock_requests_post, _mock_requests_request
+):
+    result = run_cli_dev()
+
+    assert 1 == result.exit_code
+    assert result.exception
+    assert "Unable to communicate with Keycloak API" in str(result.exception)
+    assert "Status code: 403" in str(result.exception)
+
+
+@patch(
+    "_nebari.keycloak.requests.post", side_effect=requests.exceptions.RequestException()
+)
+def test_cli_dev_keycloakapi_request_exception(_mock_requests_post):
+    result = run_cli_dev()
+
+    assert 1 == result.exit_code
+    assert result.exception
+
+
+@patch("_nebari.keycloak.requests.post", side_effect=Exception())
+def test_cli_dev_keycloakapi_unhandled_error(_mock_requests_post):
+    result = run_cli_dev()
+
+    assert 1 == result.exit_code
+    assert result.exception
+
+
+def run_cli_dev(
+    request: str = TEST_KEYCLOAKAPI_REQUEST,
+    use_env: bool = True,
+    extra_args: List[str] = [],
+):
+    with tempfile.TemporaryDirectory() as tmp:
+        tmp_file = Path(tmp).resolve() / "nebari-config.yaml"
+        assert tmp_file.exists() is False
+
+        extra_config = (
+            {
+                "domain": TEST_DOMAIN,
+                "security": {
+                    "keycloak": {
+                        "initial_root_password": MOCK_KEYCLOAK_ENV[
+                            "KEYCLOAK_ADMIN_PASSWORD"
+                        ]
+                    }
+                },
+            }
+            if not use_env
+            else {}
+        )
+        config = {**{"project_name": "dev"}, **extra_config}
+        with open(tmp_file.resolve(), "w") as f:
+            yaml.dump(config, f)
+
+        assert tmp_file.exists() is True
+
+        app = create_cli()
+
+        args = [
+            "dev",
+            "keycloak-api",
+            "--config",
+            tmp_file.resolve(),
+            "--request",
+            request,
+        ] + extra_args
+
+        env = MOCK_KEYCLOAK_ENV if use_env else {}
+        result = runner.invoke(app, args=args, env=env)
+
+        return result

tests/tests_unit/test_cli_init.py

@@ -14,30 +14,6 @@
 
 TEST_KUBERNETES_VERSION = {"default": "1.20", "do": "1.20.2-do.0"}
 
-MOCK_ENV = {
-    k: "test"
-    for k in [
-        "AWS_ACCESS_KEY_ID",
-        "AWS_SECRET_ACCESS_KEY",  # aws
-        "GOOGLE_CREDENTIALS",
-        "PROJECT_ID",  # gcp
-        "ARM_SUBSCRIPTION_ID",
-        "ARM_TENANT_ID",
-        "ARM_CLIENT_ID",
-        "ARM_CLIENT_SECRET",  # azure
-        "DIGITALOCEAN_TOKEN",
-        "SPACES_ACCESS_KEY_ID",
-        "SPACES_SECRET_ACCESS_KEY",  # digital ocean
-        "GITHUB_CLIENT_ID",
-        "GITHUB_CLIENT_SECRET",
-        "GITHUB_USERNAME",
-        "GITHUB_TOKEN",  # github
-        "AUTH0_CLIENT_ID",
-        "AUTH0_CLIENT_SECRET",
-        "AUTH0_DOMAIN",  # auth0
-    ]
-}
-
 
 @pytest.mark.parametrize(
     "args, exit_code, content",
@@ -64,17 +40,17 @@
         (["-o"], 2, ["requires an argument"]),
     ],
 )
-def test_init_stdout(args: List[str], exit_code: int, content: List[str]):
+def test_cli_init_stdout(args: List[str], exit_code: int, content: List[str]):
     app = create_cli()
     result = runner.invoke(app, ["init"] + args)
     assert result.exit_code == exit_code
     for c in content:
         assert c in result.stdout
 
 
-def generate_test_data_test_all_init_happy_path():
+def generate_test_data_test_cli_init_happy_path():
     """
-    Generate inputs to test_all_init_happy_path representing all valid combinations of options
+    Generate inputs to test_cli_init_happy_path representing all valid combinations of options
     available to nebari init
     """
 
@@ -125,7 +101,7 @@ def generate_test_data_test_all_init_happy_path():
     return {"keys": keys, "test_data": test_data}
 
 
-def test_all_init_happy_path(
+def test_cli_init_happy_path(
     provider: str,
     project_name: str,
     domain_name: str,
@@ -201,15 +177,11 @@ def assert_nebari_init_args(
     """
     with tempfile.TemporaryDirectory() as tmp:
         tmp_file = Path(tmp).resolve() / "nebari-config.yaml"
-        print(f"\n>>>> Using tmp file {tmp_file}")
         assert tmp_file.exists() is False
 
-        print(f"\n>>>> Testing nebari {args} -- input {input}")
-
         result = runner.invoke(
-            app, args + ["--output", tmp_file.resolve()], input=input, env=MOCK_ENV
+            app, args + ["--output", tmp_file.resolve()], input=input
         )
-        print(f"\n>>> runner.stdout == {result.stdout}")
 
         assert not result.exception
         assert 0 == result.exit_code