diff --git a/src/_nebari/config.py b/src/_nebari/config.py index 00c967936..d1b2f4294 100644 --- a/src/_nebari/config.py +++ b/src/_nebari/config.py @@ -115,4 +115,3 @@ def backup_configuration(filename: pathlib.Path, extrasuffix: str = ""): i = i + 1 filename.rename(backup_filename) - print(f"Backing up {filename} as {backup_filename}") diff --git a/src/_nebari/constants.py b/src/_nebari/constants.py index edc313e67..d5d20d306 100644 --- a/src/_nebari/constants.py +++ b/src/_nebari/constants.py @@ -8,7 +8,7 @@ # 04-kubernetes-ingress DEFAULT_TRAEFIK_IMAGE_TAG = "2.9.1" -HIGHEST_SUPPORTED_K8S_VERSION = ("1", "26", "7") +HIGHEST_SUPPORTED_K8S_VERSION = ("1", "26", "9") DEFAULT_GKE_RELEASE_CHANNEL = "UNSPECIFIED" DEFAULT_NEBARI_DASK_VERSION = CURRENT_RELEASE diff --git a/src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/main.tf index f16bae35e..c4b18f32a 100644 --- a/src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/main.tf +++ b/src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/main.tf @@ -57,10 +57,6 @@ resource "google_container_cluster" "main" { } } - cost_management_config { - enabled = true - } - lifecycle { ignore_changes = [ node_locations diff --git a/src/_nebari/stages/infrastructure/template/gcp/versions.tf b/src/_nebari/stages/infrastructure/template/gcp/versions.tf index 05e391e97..ddea3c185 100644 --- a/src/_nebari/stages/infrastructure/template/gcp/versions.tf +++ b/src/_nebari/stages/infrastructure/template/gcp/versions.tf @@ -2,7 +2,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = "4.83.0" + version = "4.8.0" } } required_version = ">= 1.0" diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index c9ceae4d7..fba60767e 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -11,9 +11,14 @@ from rich.prompt import Prompt from _nebari.config import backup_configuration -from _nebari.utils import load_yaml, yaml +from _nebari.utils import ( + get_k8s_version_prefix, + get_provider_config_block_name, + load_yaml, + yaml, +) from _nebari.version import __version__, rounded_ver_parse -from nebari import schema +from nebari.schema import ProviderEnum, is_version_accepted logger = logging.getLogger(__name__) @@ -22,6 +27,9 @@ ) ARGO_JUPYTER_SCHEDULER_REPO = "https://github.com/nebari-dev/argo-jupyter-scheduler" +UPGRADE_KUBERNETES_MESSAGE = "Please see the [green][link=https://www.nebari.dev/docs/how-tos/kubernetes-version-upgrade]Kubernetes upgrade docs[/link][/green] for more information." +DESTRUCTIVE_UPGRADE_WARNING = "-> This version upgrade will result in your cluster being completely torn down and redeployed. Please ensure you have backed up any data you wish to keep before proceeding!!!" + def do_upgrade(config_filename, attempt_fixes=False): config = load_yaml(config_filename) @@ -40,7 +48,7 @@ def do_upgrade(config_filename, attempt_fixes=False): ) return except (ValidationError, ValueError) as e: - if schema.is_version_accepted(config.get("nebari_version", "")): + if is_version_accepted(config.get("nebari_version", "")): # There is an unrelated validation problem rich.print( f"Your config file [purple]{config_filename}[/purple] appears to be already up-to-date for Nebari version [green]{__version__}[/green] but there is another validation error.\n" @@ -48,7 +56,6 @@ def do_upgrade(config_filename, attempt_fixes=False): raise e start_version = config.get("nebari_version", "") - print("start_version: ", start_version) UpgradeStep.upgrade( config, start_version, __version__, config_filename, attempt_fixes @@ -98,7 +105,6 @@ def upgrade( """ starting_ver = rounded_ver_parse(start_version or "0.0.0") finish_ver = rounded_ver_parse(finish_version) - print("finish_ver: ", finish_ver) if finish_ver < starting_ver: raise ValueError( @@ -116,8 +122,6 @@ def upgrade( key=rounded_ver_parse, ) - print("step_versions: ", step_versions) - current_start_version = start_version for stepcls in [cls._steps[str(v)] for v in step_versions]: step = stepcls() @@ -484,6 +488,24 @@ def _version_specific_upgrade( return config +class Upgrade_2023_7_1(UpgradeStep): + version = "2023.7.1" + + def _version_specific_upgrade( + self, config, start_version, config_filename: Path, *args, **kwargs + ): + provider = config["provider"] + if provider == ProviderEnum.aws.value: + rich.print("\n ⚠️ DANGER ⚠️") + rich.print( + DESTRUCTIVE_UPGRADE_WARNING, + "The 'prevent_deploy' flag has been set in your config file and must be manually removed to deploy.", + ) + config["prevent_deploy"] = True + + return config + + class Upgrade_2023_7_2(UpgradeStep): version = "2023.7.2" @@ -507,6 +529,107 @@ def _version_specific_upgrade( return config +class Upgrade_2023_9_1(UpgradeStep): + version = "2023.9.1" + # JupyterHub Helm chart 2.0.0 (app version 3.0.0) requires K8S Version >=1.23. (reference: https://z2jh.jupyter.org/en/stable/) + # This released has been tested against 1.26 + min_k8s_version = 1.26 + + def _version_specific_upgrade( + self, config, start_version, config_filename: Path, *args, **kwargs + ): + # Upgrading to 2023.9.1 is considered high-risk because it includes a major refacto + # to introduce the extension mechanism system. + rich.print("\n ⚠️ Warning ⚠️") + rich.print( + f"-> Nebari version [green]{self.version}[/green] includes a major refactor to introduce an extension mechanism that supports the development of third-party plugins." + ) + rich.print( + "-> Data should be backed up before performing this upgrade ([green][link=https://www.nebari.dev/docs/how-tos/manual-backup]see docs[/link][/green]) The 'prevent_deploy' flag has been set in your config file and must be manually removed to deploy." + ) + rich.print( + "-> Please also run the [green]rm -rf stages[/green] so that we can regenerate an updated set of Terraform scripts for your deployment." + ) + + # Setting the following flag will prevent deployment and display guidance to the user + # which they can override if they are happy they understand the situation. + config["prevent_deploy"] = True + + # Nebari version 2023.9.1 upgrades JupyterHub to 3.1. CDS Dashboards are only compatible with + # JupyterHub versions 1.X and so will be removed during upgrade. + rich.print("\n ⚠️ Deprecation Warning ⚠️") + rich.print( + f"-> CDS dashboards are no longer supported in Nebari version [green]{self.version}[/green] and will be uninstalled." + ) + if config.get("cdsdashboards"): + rich.print("-> Removing cdsdashboards from config file.") + del config["cdsdashboards"] + + # Kubernetes version check + # JupyterHub Helm chart 2.0.0 (app version 3.0.0) requires K8S Version >=1.23. (reference: https://z2jh.jupyter.org/en/stable/) + + provider = config["provider"] + provider_config_block = get_provider_config_block_name(provider) + + # Get current Kubernetes version if available in config. + current_version = config.get(provider_config_block, {}).get( + "kubernetes_version", None + ) + + # Convert to decimal prefix + if provider in ["aws", "azure", "gcp", "do"]: + current_version = get_k8s_version_prefix(current_version) + + # Try to convert known Kubernetes versions to float. + if current_version is not None: + try: + current_version = float(current_version) + except ValueError: + current_version = None + + # Handle checks for when Kubernetes version should be detectable + if provider in ["aws", "azure", "gcp", "do"]: + # Kubernetes version not found in provider block + if current_version is None: + rich.print("\n ⚠️ Warning ⚠️") + rich.print( + f"-> Unable to detect Kubernetes version for provider {provider}. Nebari version [green]{self.version}[/green] requires Kubernetes version {str(self.min_k8s_version)}. Please confirm your Kubernetes version is configured before upgrading." + ) + + # Kubernetes version less than required minimum + if ( + isinstance(current_version, float) + and current_version < self.min_k8s_version + ): + rich.print("\n ⚠️ Warning ⚠️") + rich.print( + f"-> Nebari version [green]{self.version}[/green] requires Kubernetes version {str(self.min_k8s_version)}. Your configured Kubernetes version is [red]{current_version}[/red]. {UPGRADE_KUBERNETES_MESSAGE}" + ) + version_diff = round(self.min_k8s_version - current_version, 2) + if version_diff > 0.01: + rich.print( + "-> The Kubernetes version is multiple minor versions behind the minimum required version. You will need to perform the upgrade one minor version at a time. For example, if your current version is 1.24, you will need to upgrade to 1.25, and then 1.26." + ) + rich.print( + f"-> Update the value of [green]{provider_config_block}.kubernetes_version[/green] in your config file to a newer version of Kubernetes and redeploy." + ) + + else: + rich.print("\n ⚠️ Warning ⚠️") + rich.print( + f"-> Unable to detect Kubernetes version for provider {provider}. Nebari version [green]{self.version}[/green] requires Kubernetes version {str(self.min_k8s_version)} or greater." + ) + rich.print( + "-> Please ensure your Kubernetes version is up-to-date before proceeding." + ) + + if provider == "aws": + rich.print("\n ⚠️ DANGER ⚠️") + rich.print(DESTRUCTIVE_UPGRADE_WARNING) + + return config + + __rounded_version__ = ".".join([str(c) for c in rounded_ver_parse(__version__)]) # Manually-added upgrade steps must go above this line diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index 77f6f1016..3378116a1 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -314,3 +314,39 @@ def construct_azure_resource_group_name( if base_resource_group_name: return f"{base_resource_group_name}{suffix}" return f"{project_name}-{namespace}{suffix}" + + +def get_k8s_version_prefix(k8s_version: str) -> str: + """Return the major.minor version of the k8s version string.""" + + k8s_version = str(k8s_version) + # Split the input string by the first decimal point + parts = k8s_version.split(".", 1) + + if len(parts) == 2: + # Extract the part before the second decimal point + before_second_decimal = parts[0] + "." + parts[1].split(".")[0] + try: + # Convert the extracted part to a float + result = float(before_second_decimal) + return result + except ValueError: + # Handle the case where the conversion to float fails + return None + else: + # Handle the case where there is no second decimal point + return None + + +def get_provider_config_block_name(provider): + PROVIDER_CONFIG_NAMES = { + "aws": "amazon_web_services", + "azure": "azure", + "do": "digital_ocean", + "gcp": "google_cloud_platform", + } + + if provider in PROVIDER_CONFIG_NAMES.keys(): + return PROVIDER_CONFIG_NAMES[provider] + else: + return provider diff --git a/tests/tests_unit/test_cli_upgrade.py b/tests/tests_unit/test_cli_upgrade.py index 0ffe37000..e0fe685af 100644 --- a/tests/tests_unit/test_cli_upgrade.py +++ b/tests/tests_unit/test_cli_upgrade.py @@ -1,3 +1,4 @@ +import re import tempfile from pathlib import Path from typing import Any, Dict, List @@ -9,6 +10,22 @@ import _nebari.upgrade import _nebari.version from _nebari.cli import create_cli +from _nebari.constants import AZURE_DEFAULT_REGION +from _nebari.upgrade import UPGRADE_KUBERNETES_MESSAGE +from _nebari.utils import get_provider_config_block_name + +MOCK_KUBERNETES_VERSIONS = { + "aws": ["1.20"], + "azure": ["1.20"], + "gcp": ["1.20"], + "do": ["1.21.5-do.0"], +} +MOCK_CLOUD_REGIONS = { + "aws": ["us-east-1"], + "azure": [AZURE_DEFAULT_REGION], + "gcp": ["us-central1"], + "do": ["nyc3"], +} # can't upgrade to a previous version that doesn't have a corresponding @@ -87,11 +104,28 @@ def test_cli_upgrade_2023_4_1_to_2023_5_1(monkeypatch: pytest.MonkeyPatch): ) +@pytest.mark.parametrize( + "provider", + ["aws", "azure", "do", "gcp"], +) +def test_cli_upgrade_2023_5_1_to_2023_7_1( + monkeypatch: pytest.MonkeyPatch, provider: str +): + config = assert_nebari_upgrade_success( + monkeypatch, "2023.5.1", "2023.7.1", provider=provider + ) + prevent_deploy = config.get("prevent_deploy") + if provider == "aws": + assert prevent_deploy + else: + assert not prevent_deploy + + @pytest.mark.parametrize( "workflows_enabled, workflow_controller_enabled", [(True, True), (True, False), (False, None), (None, None)], ) -def test_cli_upgrade_2023_5_1_to_2023_7_2( +def test_cli_upgrade_2023_7_1_to_2023_7_2( monkeypatch: pytest.MonkeyPatch, workflows_enabled: bool, workflow_controller_enabled: bool, @@ -106,7 +140,7 @@ def test_cli_upgrade_2023_5_1_to_2023_7_2( upgraded = assert_nebari_upgrade_success( monkeypatch, - "2023.5.1", + "2023.7.1", "2023.7.2", addl_config=addl_config, # Do you want to enable the Nebari Workflow Controller? @@ -132,7 +166,7 @@ def test_cli_upgrade_2023_5_1_to_2023_7_2( def test_cli_upgrade_image_tags(monkeypatch: pytest.MonkeyPatch): start_version = "2023.5.1" - end_version = "2023.7.2" + end_version = "2023.7.1" upgraded = assert_nebari_upgrade_success( monkeypatch, @@ -383,10 +417,134 @@ def test_cli_upgrade_to_0_4_0_fails_for_custom_auth_without_attempt_fixes(): assert yaml.safe_load(c) == nebari_config +@pytest.mark.skipif( + _nebari.upgrade.__version__ < "2023.9.1", + reason="This test is only valid for versions <= 2023.9.1", +) +def test_cli_upgrade_to_2023_9_1_cdsdashboard_removed(monkeypatch: pytest.MonkeyPatch): + start_version = "2023.7.2" + end_version = "2023.9.1" + + addl_config = yaml.safe_load( + """ +cdsdashboards: + enabled: true + cds_hide_user_named_servers: true + cds_hide_user_dashboard_servers: false + """ + ) + + upgraded = assert_nebari_upgrade_success( + monkeypatch, + start_version, + end_version, + addl_args=["--attempt-fixes"], + addl_config=addl_config, + ) + + assert not upgraded.get("cdsdashboards") + assert upgraded.get("prevent_deploy") + + +@pytest.mark.skipif( + _nebari.upgrade.__version__ < "2023.9.1", + reason="This test is only valid for versions <= 2023.9.1", +) +@pytest.mark.parametrize( + ("provider", "k8s_status"), + [ + ("aws", "compatible"), + ("aws", "incompatible"), + ("aws", "invalid"), + ("azure", "compatible"), + ("azure", "incompatible"), + ("azure", "invalid"), + ("do", "compatible"), + ("do", "incompatible"), + ("do", "invalid"), + ("gcp", "compatible"), + ("gcp", "incompatible"), + ("gcp", "invalid"), + ], +) +def test_cli_upgrade_to_2023_9_1_kubernetes_validations( + monkeypatch: pytest.MonkeyPatch, provider: str, k8s_status: str +): + start_version = "2023.7.2" + end_version = "2023.9.1" + monkeypatch.setattr(_nebari.upgrade, "__version__", end_version) + + kubernetes_configs = { + "aws": {"incompatible": "1.19", "compatible": "1.26", "invalid": "badname"}, + "azure": {"incompatible": "1.23", "compatible": "1.26", "invalid": "badname"}, + "do": { + "incompatible": "1.19.2-do.3", + "compatible": "1.26.0-do.custom", + "invalid": "badname", + }, + "gcp": {"incompatible": "1.23", "compatible": "1.26", "invalid": "badname"}, + } + + with tempfile.TemporaryDirectory() as tmp: + tmp_file = Path(tmp).resolve() / "nebari-config.yaml" + assert tmp_file.exists() is False + + nebari_config = yaml.safe_load( + f""" +project_name: test +provider: {provider} +domain: test.example.com +namespace: dev +nebari_version: {start_version} +cdsdashboards: + enabled: true + cds_hide_user_named_servers: true + cds_hide_user_dashboard_servers: false +{get_provider_config_block_name(provider)}: + region: {MOCK_CLOUD_REGIONS.get(provider, {})[0]} + kubernetes_version: {kubernetes_configs[provider][k8s_status]} + """ + ) + with open(tmp_file.resolve(), "w") as f: + yaml.dump(nebari_config, f) + + assert tmp_file.exists() is True + app = create_cli() + + result = runner.invoke(app, ["upgrade", "--config", tmp_file.resolve()]) + + if k8s_status == "incompatible": + UPGRADE_KUBERNETES_MESSAGE_WO_BRACKETS = re.sub( + r"\[.*?\]", "", UPGRADE_KUBERNETES_MESSAGE + ) + assert UPGRADE_KUBERNETES_MESSAGE_WO_BRACKETS in result.stdout.replace( + "\n", "" + ) + + if k8s_status == "compatible": + assert 0 == result.exit_code + assert not result.exception + assert "Saving new config file" in result.stdout + + # load the modified nebari-config.yaml and check the new version has changed + with open(tmp_file.resolve(), "r") as f: + upgraded = yaml.safe_load(f) + assert end_version == upgraded["nebari_version"] + + if k8s_status == "invalid": + assert ( + "Unable to detect Kubernetes version for provider {}".format( + provider + ) + in result.stdout + ) + + def assert_nebari_upgrade_success( monkeypatch: pytest.MonkeyPatch, start_version: str, end_version: str, + provider: str = "local", addl_args: List[str] = [], addl_config: Dict[str, Any] = {}, inputs: List[str] = [], @@ -404,7 +562,7 @@ def assert_nebari_upgrade_success( **yaml.safe_load( f""" project_name: test -provider: local +provider: {provider} domain: test.example.com namespace: dev nebari_version: {start_version} diff --git a/tests/tests_unit/test_cli_validate.py b/tests/tests_unit/test_cli_validate.py index 13955c1fc..0894da927 100644 --- a/tests/tests_unit/test_cli_validate.py +++ b/tests/tests_unit/test_cli_validate.py @@ -7,6 +7,7 @@ import yaml from typer.testing import CliRunner +from _nebari._version import __version__ from _nebari.cli import create_cli TEST_DATA_DIR = Path(__file__).resolve().parent / "cli_validate" @@ -14,6 +15,17 @@ runner = CliRunner() +def _update_yaml_file(file_path: Path, key: str, value: Any): + """Utility function to update a yaml file with a new key/value pair.""" + with open(file_path, "r") as f: + yaml_data = yaml.safe_load(f) + + yaml_data[key] = value + + with open(file_path, "w") as f: + yaml.safe_dump(yaml_data, f) + + @pytest.mark.parametrize( "args, exit_code, content", [ @@ -62,6 +74,9 @@ def test_cli_validate_local_happy_path(config_yaml: str): test_file = TEST_DATA_DIR / config_yaml assert test_file.exists() is True + # update the test file with the current version + _update_yaml_file(test_file, "nebari_version", __version__) + app = create_cli() result = runner.invoke(app, ["validate", "--config", test_file]) assert not result.exception