From 624dad90a8a323295224cbe106f3b401ba25de77 Mon Sep 17 00:00:00 2001 From: Ken Foster Date: Tue, 19 Sep 2023 21:03:20 +0000 Subject: [PATCH 01/17] Upgrade step - remove CDS dashboard config --- src/_nebari/upgrade.py | 27 +++++++++++++++++++++++++++ tests/tests_unit/test_cli_upgrade.py | 26 ++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index c9ceae4d73..0141064b9f 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -506,6 +506,33 @@ def _version_specific_upgrade( return config +class Upgrade_2023_9_1(UpgradeStep): + version = "2023.9.1" + + def _version_specific_upgrade( + self, config, start_version, config_filename: Path, *args, **kwargs + ): + # It is not safe to immediately redeploy without backing up data ready to restore data + # since a new cluster will be created for the new version. + # Setting the following flag will prevent deployment and display guidance to the user + # which they can override if they are happy they understand the situation. + config["prevent_deploy"] = True + + + # Nebari version 2023.9.1 upgrades JupyterHub to 3.1. CDS Dashboards are only compatible with + # JupyterHub versions 1.X and so will be removed during upgrade. + rich.print("\n ⚠️ Deprecation Warning ⚠️") + rich.print( + f"-> CDS dashboards are no longer supported in Nebari version [green]{self.version}[/green] and will be uninstalled." + ) + if config.get("cdsdashboards"): + rich.print( + f"-> Removing cdsdashboards from config file." + ) + del config["cdsdashboards"] + + return config + __rounded_version__ = ".".join([str(c) for c in rounded_ver_parse(__version__)]) diff --git a/tests/tests_unit/test_cli_upgrade.py b/tests/tests_unit/test_cli_upgrade.py index 0ffe370009..cc0d380929 100644 --- a/tests/tests_unit/test_cli_upgrade.py +++ b/tests/tests_unit/test_cli_upgrade.py @@ -382,6 +382,32 @@ def test_cli_upgrade_to_0_4_0_fails_for_custom_auth_without_attempt_fixes(): with open(tmp_file.resolve(), "r") as c: assert yaml.safe_load(c) == nebari_config +def test_cli_upgrade_to_2023_9_1_cdsdashboard_removed(monkeypatch: pytest.MonkeyPatch): + start_version = "2023.5.2" + end_version = "2023.9.1" + + with tempfile.TemporaryDirectory() as tmp: + tmp_file = Path(tmp).resolve() / "nebari-config.yaml" + assert tmp_file.exists() is False + + addl_config = yaml.safe_load( + f""" +cdsdashboards: + enabled: true + cds_hide_user_named_servers: true + cds_hide_user_dashboard_servers: false + """ + ) + + upgraded = assert_nebari_upgrade_success( + monkeypatch, + start_version, + end_version, + addl_args=["--attempt-fixes"], + addl_config=addl_config + ) + + assert not upgraded.get("cdsdashboards") def assert_nebari_upgrade_success( monkeypatch: pytest.MonkeyPatch, From 24ceff4952891a2a83fb4c15fbc1df2f7000a3ed Mon Sep 17 00:00:00 2001 From: Ken Foster Date: Tue, 19 Sep 2023 21:54:16 +0000 Subject: [PATCH 02/17] start K8S check action --- src/_nebari/upgrade.py | 81 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 3 deletions(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 0141064b9f..361260590c 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -11,8 +11,14 @@ from rich.prompt import Prompt from _nebari.config import backup_configuration -from _nebari.utils import load_yaml, yaml +from _nebari.utils import load_yaml, get_latest_kubernetes_version, yaml from _nebari.version import __version__, rounded_ver_parse +from _nebari.provider.cloud import ( + amazon_web_services, + azure_cloud, + digital_ocean, + google_cloud, +) from nebari import schema logger = logging.getLogger(__name__) @@ -512,8 +518,15 @@ class Upgrade_2023_9_1(UpgradeStep): def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): - # It is not safe to immediately redeploy without backing up data ready to restore data - # since a new cluster will be created for the new version. + # Upgrading to 2023.9.1 is considered high-risk because it includes a major refacto + # to introduce the extension mechanism system. + rich.print("\n ⚠️ Warning ⚠️") + rich.print( + f"-> Nebari version [green]{self.version}[/green] includes a major refactor to introduce an extension mechanism that supports the development of third-party plugins." + ) + rich.print("-> Data should be backed up before performing this upgrade. The 'prevent_deploy' flag has been set in your config file and must be manually removed to deploy." + ) + # Setting the following flag will prevent deployment and display guidance to the user # which they can override if they are happy they understand the situation. config["prevent_deploy"] = True @@ -531,6 +544,68 @@ def _version_specific_upgrade( ) del config["cdsdashboards"] + # Kubernetes version check. JupyterHub Helm chart 2.0.0 (app version 3.0.0) requires K8S Version >=1.23. (reference: https://z2jh.jupyter.org/en/stable/) + match config["provider"]: + case "aws": + current_version = float(config["amazon_web_services"]["kubernetes_version"]) + available_version = get_latest_kubernetes_version(amazon_web_services.kubernetes_versions(config["region"])) + + if current_version < 1.23: + rich.print( + f"-> Nebari version [green]{self.version}[/green] requires Kubernetes version 1.23 or greater. Your configured Kubernetes version is [red]{current_version}[/red]." + ) + rich.print( + f"-> Please upgrade your EKS cluster outside of Nebari following Amazon Web Services guide: https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html. We recommend upgrading to the latest available version in your region, which is [red]{available_version}[/red]" + ) + rich.print( + f"-> Once you have completed the upgrade, update the value of amazon_web_services.kubernetes_version in your config file to match and run the update command again." + ) + exit() + + case "azure": + current_version = float(config["azure_cloud"]["kubernetes_version"]) + available_version = get_latest_kubernetes_version(azure_cloud.kubernetes_versions(config["region"])) + + if current_version < 1.23: + rich.print( + f"-> Nebari version [green]{self.version}[/green] requires Kubernetes version 1.23 or greater. Your configured Kubernetes version is [red]{current_version}[/red]." + ) + rich.print( + f"-> Please upgrade your AKS cluster outside of Nebari following Azure Cloud's guide: https://learn.microsoft.com/en-us/azure/aks/upgrade-cluster. We recommend upgrading to the latest available version in your region, which is [red]{available_version}[/red]" + ) + rich.print( + f"-> Once you have completed the upgrade, update the value of azure_cloud.kubernetes_version in your config file to match and run the update command again." + ) + exit() + + case "gcp": + current_version = float(config["google_cloud"]["kubernetes_version"]) + available_version = get_latest_kubernetes_version(google_cloud.kubernetes_versions(config["region"])) + + if current_version < 1.23: + rich.print( + f"-> Nebari version [green]{self.version}[/green] requires Kubernetes version 1.23 or greater. Your configured Kubernetes version is [red]{current_version}[/red]." + ) + rich.print( + f"-> Please upgrade your GKE cluster outside of Nebari following Azure Cloud's guide: https://cloud.google.com/kubernetes-engine/docs/how-to/upgrading-a-cluster. We recommend upgrading to the latest available version in your region, which is [red]{available_version}[/red]" + ) + rich.print( + f"-> Once you have completed the upgrade, update the value of google_cloud.kubernetes_version in your config file to match and run the update command again." + ) + exit() + + case "do": + # TODO: DO kubernetes_version strings are not floats. Need to confirm if truncating (e.g. '1.21.5-do.0' to '1.21') is valid. + pass + + case "local": + # TODO: Are K8S versions set in config for local deployments? + pass + + case "existing": + # TODO: Are K8S versions set in config for existing deployments? + pass + return config From 8b61979d50e48687caacc14913d39d8d28c40fbb Mon Sep 17 00:00:00 2001 From: Ken Foster Date: Wed, 20 Sep 2023 18:31:27 +0000 Subject: [PATCH 03/17] Add K8S checks --- src/_nebari/upgrade.py | 116 +++++++++++++-------------- src/_nebari/utils.py | 33 ++++++++ tests/tests_unit/test_cli_upgrade.py | 89 ++++++++++++++++---- 3 files changed, 161 insertions(+), 77 deletions(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 361260590c..30ce89b888 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -11,14 +11,8 @@ from rich.prompt import Prompt from _nebari.config import backup_configuration -from _nebari.utils import load_yaml, get_latest_kubernetes_version, yaml +from _nebari.utils import load_yaml, get_latest_kubernetes_version, get_do_k8s_version_prefix, get_provider_config_block_name, yaml from _nebari.version import __version__, rounded_ver_parse -from _nebari.provider.cloud import ( - amazon_web_services, - azure_cloud, - digital_ocean, - google_cloud, -) from nebari import schema logger = logging.getLogger(__name__) @@ -544,68 +538,68 @@ def _version_specific_upgrade( ) del config["cdsdashboards"] - # Kubernetes version check. JupyterHub Helm chart 2.0.0 (app version 3.0.0) requires K8S Version >=1.23. (reference: https://z2jh.jupyter.org/en/stable/) - match config["provider"]: - case "aws": - current_version = float(config["amazon_web_services"]["kubernetes_version"]) - available_version = get_latest_kubernetes_version(amazon_web_services.kubernetes_versions(config["region"])) - - if current_version < 1.23: - rich.print( - f"-> Nebari version [green]{self.version}[/green] requires Kubernetes version 1.23 or greater. Your configured Kubernetes version is [red]{current_version}[/red]." - ) - rich.print( - f"-> Please upgrade your EKS cluster outside of Nebari following Amazon Web Services guide: https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html. We recommend upgrading to the latest available version in your region, which is [red]{available_version}[/red]" - ) - rich.print( - f"-> Once you have completed the upgrade, update the value of amazon_web_services.kubernetes_version in your config file to match and run the update command again." - ) - exit() - - case "azure": - current_version = float(config["azure_cloud"]["kubernetes_version"]) - available_version = get_latest_kubernetes_version(azure_cloud.kubernetes_versions(config["region"])) - - if current_version < 1.23: - rich.print( - f"-> Nebari version [green]{self.version}[/green] requires Kubernetes version 1.23 or greater. Your configured Kubernetes version is [red]{current_version}[/red]." - ) - rich.print( - f"-> Please upgrade your AKS cluster outside of Nebari following Azure Cloud's guide: https://learn.microsoft.com/en-us/azure/aks/upgrade-cluster. We recommend upgrading to the latest available version in your region, which is [red]{available_version}[/red]" - ) - rich.print( - f"-> Once you have completed the upgrade, update the value of azure_cloud.kubernetes_version in your config file to match and run the update command again." - ) - exit() + # Kubernetes version check. Minimum Kubernetes version is 1.26 + # JupyterHub Helm chart 2.0.0 (app version 3.0.0) requires K8S Version >=1.23. (reference: https://z2jh.jupyter.org/en/stable/) + + # Upgrade instructions for various providers + upgrade_messages = { + "aws": "Please upgrade your EKS cluster outside of Nebari following Amazon Web Services guide: https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html.", + "azure": "Please upgrade your AKS cluster outside of Nebari following Azure Cloud's guide: https://learn.microsoft.com/en-us/azure/aks/upgrade-cluster.", + "gcp": "Please upgrade your GKE cluster outside of Nebari following Google Cloud Platform's guide: https://cloud.google.com/kubernetes-engine/docs/how-to/upgrading-a-cluster.", + "do": "Please upgrade your DOKS cluster outside of Nebari following Digital Ocean's guide: https://docs.digitalocean.com/products/kubernetes/how-to/upgrade-cluster/" + } + + provider = config["provider"] + provider_config_block = get_provider_config_block_name(provider) + + # Get current Kubernetes version if available in config. + current_version = config.get(provider_config_block, {}).get("kubernetes_version", "NA") + + # Convert to decimal prefix if provider is Digital Ocean + if provider == "do": + current_version = get_do_k8s_version_prefix(current_version) + rich.print(current_version) + + # Try to convert known Kubernetes versions to float. + if not current_version == "NA": + try: + current_version = float(current_version) + except: + current_version = "NA" + + # Handle checks for when Kubernetes version should be detectable + if provider in ["aws","azure","gcp","do"]: - case "gcp": - current_version = float(config["google_cloud"]["kubernetes_version"]) - available_version = get_latest_kubernetes_version(google_cloud.kubernetes_versions(config["region"])) - - if current_version < 1.23: + # Kubernetes version not found in provider block + if current_version == "NA": + rich.print("\n ⚠️ Warning ⚠️") + rich.print( + f"-> Unable to detect Kubernetes version for provider {provider}. Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26 or greater. Please confirm your Kubernetes version is configured before upgrading." + ) + exit() + + # Kubernetes version less than required minimum + if current_version < 1.26: + rich.print("\n ⚠️ Warning ⚠️") rich.print( - f"-> Nebari version [green]{self.version}[/green] requires Kubernetes version 1.23 or greater. Your configured Kubernetes version is [red]{current_version}[/red]." + f"-> Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26 or greater. Your configured Kubernetes version is [red]{current_version}[/red]." ) rich.print( - f"-> Please upgrade your GKE cluster outside of Nebari following Azure Cloud's guide: https://cloud.google.com/kubernetes-engine/docs/how-to/upgrading-a-cluster. We recommend upgrading to the latest available version in your region, which is [red]{available_version}[/red]" + upgrade_messages[provider] ) rich.print( - f"-> Once you have completed the upgrade, update the value of google_cloud.kubernetes_version in your config file to match and run the update command again." + f"-> Once you have completed the upgrade, update the value of {provider_config_block}.kubernetes_version in your config file to match and run the upgrade command again." ) exit() - - case "do": - # TODO: DO kubernetes_version strings are not floats. Need to confirm if truncating (e.g. '1.21.5-do.0' to '1.21') is valid. - pass - - case "local": - # TODO: Are K8S versions set in config for local deployments? - pass - - case "existing": - # TODO: Are K8S versions set in config for existing deployments? - pass - + + else: + rich.print("\n ⚠️ Warning ⚠️") + rich.print( + f"-> Unable to detect Kubernetes version for provider {provider}. Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26 or greater." + ) + rich.print( + f"-> Please ensure your Kubernetes version is up-to-date before proceeding." + ) return config diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index 77f6f10160..977a3084d0 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -314,3 +314,36 @@ def construct_azure_resource_group_name( if base_resource_group_name: return f"{base_resource_group_name}{suffix}" return f"{project_name}-{namespace}{suffix}" + +def get_do_k8s_version_prefix(do_k8s_version): + # Split the input string by the first decimal point + parts = do_k8s_version.split('.', 1) + + if len(parts) == 2: + # Extract the part before the second decimal point + before_second_decimal = parts[0] + '.' + parts[1].split('.')[0] + + try: + # Convert the extracted part to a float + result = float(before_second_decimal) + return result + except ValueError: + # Handle the case where the conversion to float fails + return None + else: + # Handle the case where there is no second decimal point + return None + +def get_provider_config_block_name(provider): + + PROVIDER_CONFIG_NAMES = { + "aws": "amazon_web_services", + "azure": "azure_cloud", + "do": "digital_ocean", + "gcp": "google_cloud" + } + + if provider in PROVIDER_CONFIG_NAMES.keys(): + return PROVIDER_CONFIG_NAMES[provider] + else: + return provider \ No newline at end of file diff --git a/tests/tests_unit/test_cli_upgrade.py b/tests/tests_unit/test_cli_upgrade.py index cc0d380929..ab1b5b388d 100644 --- a/tests/tests_unit/test_cli_upgrade.py +++ b/tests/tests_unit/test_cli_upgrade.py @@ -8,8 +8,23 @@ import _nebari.upgrade import _nebari.version +from _nebari.utils import get_provider_config_block_name from _nebari.cli import create_cli - +from _nebari.constants import AZURE_DEFAULT_REGION + + +MOCK_KUBERNETES_VERSIONS = { + "aws": ["1.20"], + "azure": ["1.20"], + "gcp": ["1.20"], + "do": ["1.21.5-do.0"], +} +MOCK_CLOUD_REGIONS = { + "aws": ["us-east-1"], + "azure": [AZURE_DEFAULT_REGION], + "gcp": ["us-central1"], + "do": ["nyc3"], +} # can't upgrade to a previous version that doesn't have a corresponding # UpgradeStep derived class. without these dummy classes, the rendered @@ -386,28 +401,70 @@ def test_cli_upgrade_to_2023_9_1_cdsdashboard_removed(monkeypatch: pytest.Monkey start_version = "2023.5.2" end_version = "2023.9.1" - with tempfile.TemporaryDirectory() as tmp: - tmp_file = Path(tmp).resolve() / "nebari-config.yaml" - assert tmp_file.exists() is False - - addl_config = yaml.safe_load( - f""" + addl_config = yaml.safe_load( + f""" cdsdashboards: enabled: true cds_hide_user_named_servers: true cds_hide_user_dashboard_servers: false """ - ) + ) - upgraded = assert_nebari_upgrade_success( - monkeypatch, - start_version, - end_version, - addl_args=["--attempt-fixes"], - addl_config=addl_config - ) + upgraded = assert_nebari_upgrade_success( + monkeypatch, + start_version, + end_version, + addl_args=["--attempt-fixes"], + addl_config=addl_config + ) - assert not upgraded.get("cdsdashboards") + assert not upgraded.get("cdsdashboards") + assert upgraded.get("prevent_deploy") + +#def test_cli_upgrade_to_2023_9_1_kubernetes_validations(monkeypatch: pytest.MonkeyPatch): +# start_version = "2023.7.2" +# monkeypatch.setattr(_nebari.upgrade, "__version__", "2023.9.1") +# +# kubernetes_configs = { +# "aws": {"incompatible": "1.19", "compatible": "1.26", "invalid": "badname"}, +# "azure": {"incompatible": "1.23", "compatible": "1.26", "invalid": "badname"}, +# "do": {"incompatible": "1.19.2-do.3", "compatible": "1.26.0-do.custom", "invalid": "badname"}, +# "gcp": {"incompatible": "1.23", "compatible": "1.26", "invalid": "badname"}, +# } +# +# # Incompatible provider tests +# for provider in kubernetes_configs.keys(): +# +# with tempfile.TemporaryDirectory() as tmp: +# tmp_file = Path(tmp).resolve() / "nebari-config.yaml" +# assert tmp_file.exists() is False +# +# nebari_config = yaml.safe_load( +# f""" +#project_name: test +#provider: {provider} +#domain: test.example.com +#namespace: dev +#nebari_version: {start_version} +#cdsdashboards: +# enabled: true +# cds_hide_user_named_servers: true +# cds_hide_user_dashboard_servers: false +#{get_provider_config_block_name(provider)}: +## region: {MOCK_CLOUD_REGIONS.get(provider, {})[0]} +# kubernetes_version: {kubernetes_configs[provider]["incompatible"]} +# """ +# ) +# +# with open(tmp_file.resolve(), "w") as f: +# yaml.dump(nebari_config, f) +# +# assert tmp_file.exists() is True +# app = create_cli() +# +# result = runner.invoke(app, ["upgrade", "--config", tmp_file.resolve()]) +# +# assert "AWS" in result.stdout def assert_nebari_upgrade_success( monkeypatch: pytest.MonkeyPatch, From d8c9679535341608663b3c77310ec59c1b7a83d1 Mon Sep 17 00:00:00 2001 From: Ken Foster Date: Wed, 20 Sep 2023 20:37:05 +0000 Subject: [PATCH 04/17] Update K8S checks --- src/_nebari/upgrade.py | 4 +- tests/tests_unit/test_cli_upgrade.py | 131 ++++++++++++++++++--------- 2 files changed, 88 insertions(+), 47 deletions(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 30ce89b888..95f0c15aec 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -576,7 +576,7 @@ def _version_specific_upgrade( rich.print( f"-> Unable to detect Kubernetes version for provider {provider}. Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26 or greater. Please confirm your Kubernetes version is configured before upgrading." ) - exit() + exit(1) # Kubernetes version less than required minimum if current_version < 1.26: @@ -590,7 +590,7 @@ def _version_specific_upgrade( rich.print( f"-> Once you have completed the upgrade, update the value of {provider_config_block}.kubernetes_version in your config file to match and run the upgrade command again." ) - exit() + exit(1) else: rich.print("\n ⚠️ Warning ⚠️") diff --git a/tests/tests_unit/test_cli_upgrade.py b/tests/tests_unit/test_cli_upgrade.py index ab1b5b388d..4ce778a21a 100644 --- a/tests/tests_unit/test_cli_upgrade.py +++ b/tests/tests_unit/test_cli_upgrade.py @@ -398,7 +398,7 @@ def test_cli_upgrade_to_0_4_0_fails_for_custom_auth_without_attempt_fixes(): assert yaml.safe_load(c) == nebari_config def test_cli_upgrade_to_2023_9_1_cdsdashboard_removed(monkeypatch: pytest.MonkeyPatch): - start_version = "2023.5.2" + start_version = "2023.7.2" end_version = "2023.9.1" addl_config = yaml.safe_load( @@ -421,50 +421,91 @@ def test_cli_upgrade_to_2023_9_1_cdsdashboard_removed(monkeypatch: pytest.Monkey assert not upgraded.get("cdsdashboards") assert upgraded.get("prevent_deploy") -#def test_cli_upgrade_to_2023_9_1_kubernetes_validations(monkeypatch: pytest.MonkeyPatch): -# start_version = "2023.7.2" -# monkeypatch.setattr(_nebari.upgrade, "__version__", "2023.9.1") -# -# kubernetes_configs = { -# "aws": {"incompatible": "1.19", "compatible": "1.26", "invalid": "badname"}, -# "azure": {"incompatible": "1.23", "compatible": "1.26", "invalid": "badname"}, -# "do": {"incompatible": "1.19.2-do.3", "compatible": "1.26.0-do.custom", "invalid": "badname"}, -# "gcp": {"incompatible": "1.23", "compatible": "1.26", "invalid": "badname"}, -# } -# -# # Incompatible provider tests -# for provider in kubernetes_configs.keys(): -# -# with tempfile.TemporaryDirectory() as tmp: -# tmp_file = Path(tmp).resolve() / "nebari-config.yaml" -# assert tmp_file.exists() is False -# -# nebari_config = yaml.safe_load( -# f""" -#project_name: test -#provider: {provider} -#domain: test.example.com -#namespace: dev -#nebari_version: {start_version} -#cdsdashboards: -# enabled: true -# cds_hide_user_named_servers: true -# cds_hide_user_dashboard_servers: false -#{get_provider_config_block_name(provider)}: -## region: {MOCK_CLOUD_REGIONS.get(provider, {})[0]} -# kubernetes_version: {kubernetes_configs[provider]["incompatible"]} -# """ -# ) -# -# with open(tmp_file.resolve(), "w") as f: -# yaml.dump(nebari_config, f) -# -# assert tmp_file.exists() is True -# app = create_cli() -# -# result = runner.invoke(app, ["upgrade", "--config", tmp_file.resolve()]) -# -# assert "AWS" in result.stdout +@pytest.mark.parametrize( + ("provider","k8s_status"), + [ + ("aws", "compatible"), + ("aws", "incompatible"), + ("aws", "invalid"), + ("azure", "compatible"), + ("azure", "incompatible"), + ("azure", "invalid"), + ("do", "compatible"), + ("do", "incompatible"), + ("do", "invalid"), + ("gcp", "compatible"), + ("gcp", "incompatible"), + ("gcp", "invalid"), + ] +) + +def test_cli_upgrade_to_2023_9_1_kubernetes_validations(monkeypatch: pytest.MonkeyPatch, provider: str, k8s_status: str): + start_version = "2023.7.2" + end_version = "2023.9.1" + monkeypatch.setattr(_nebari.upgrade, "__version__", end_version) + + kubernetes_configs = { + "aws": {"incompatible": "1.19", "compatible": "1.26", "invalid": "badname"}, + "azure": {"incompatible": "1.23", "compatible": "1.26", "invalid": "badname"}, + "do": {"incompatible": "1.19.2-do.3", "compatible": "1.26.0-do.custom", "invalid": "badname"}, + "gcp": {"incompatible": "1.23", "compatible": "1.26", "invalid": "badname"}, + } + + # Incompatible provider tests + upgrade_messages = { + "aws": "Please upgrade your EKS cluster outside of Nebari following Amazon Web Services guide", + "azure": "Please upgrade your AKS cluster outside of Nebari following Azure Cloud's guide", + "gcp": "Please upgrade your GKE cluster outside of Nebari following Google Cloud Platform's guide", + "do": "Please upgrade your DOKS cluster outside of Nebari following Digital Ocean's guide" + } + + with tempfile.TemporaryDirectory() as tmp: + tmp_file = Path(tmp).resolve() / "nebari-config.yaml" + assert tmp_file.exists() is False + + nebari_config = yaml.safe_load( + f""" +project_name: test +provider: {provider} +domain: test.example.com +namespace: dev +nebari_version: {start_version} +cdsdashboards: + enabled: true + cds_hide_user_named_servers: true + cds_hide_user_dashboard_servers: false +{get_provider_config_block_name(provider)}: + region: {MOCK_CLOUD_REGIONS.get(provider, {})[0]} + kubernetes_version: {kubernetes_configs[provider][k8s_status]} + """ + ) + with open(tmp_file.resolve(), "w") as f: + yaml.dump(nebari_config, f) + + assert tmp_file.exists() is True + app = create_cli() + + result = runner.invoke(app, ["upgrade", "--config", tmp_file.resolve()]) + + if k8s_status == "incompatible": + assert 1 == result.exit_code + assert result.exception + assert upgrade_messages[provider] in result.stdout.replace("\n","") + + if k8s_status == "compatible": + assert 0 == result.exit_code + assert not result.exception + assert "Saving new config file" in result.stdout + + # load the modified nebari-config.yaml and check the new version has changed + with open(tmp_file.resolve(), "r") as f: + upgraded = yaml.safe_load(f) + assert end_version == upgraded["nebari_version"] + + if k8s_status == "invalid": + assert 1 == result.exit_code + assert result.exception + assert "Unable to detect Kubernetes version for provider {}".format(provider) in result.stdout def assert_nebari_upgrade_success( monkeypatch: pytest.MonkeyPatch, From 2fda6ab86acb6e1c3c34c4789a42898f226897cc Mon Sep 17 00:00:00 2001 From: Ken Foster Date: Wed, 20 Sep 2023 20:48:08 +0000 Subject: [PATCH 05/17] pre-commit changes --- src/_nebari/upgrade.py | 69 +++++++++++++------------- src/_nebari/utils.py | 13 ++--- tests/tests_unit/test_cli_upgrade.py | 73 ++++++++++++++++------------ 3 files changed, 86 insertions(+), 69 deletions(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 95f0c15aec..499be6cb1e 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -11,7 +11,12 @@ from rich.prompt import Prompt from _nebari.config import backup_configuration -from _nebari.utils import load_yaml, get_latest_kubernetes_version, get_do_k8s_version_prefix, get_provider_config_block_name, yaml +from _nebari.utils import ( + get_do_k8s_version_prefix, + get_provider_config_block_name, + load_yaml, + yaml, +) from _nebari.version import __version__, rounded_ver_parse from nebari import schema @@ -506,6 +511,7 @@ def _version_specific_upgrade( return config + class Upgrade_2023_9_1(UpgradeStep): version = "2023.9.1" @@ -517,15 +523,15 @@ def _version_specific_upgrade( rich.print("\n ⚠️ Warning ⚠️") rich.print( f"-> Nebari version [green]{self.version}[/green] includes a major refactor to introduce an extension mechanism that supports the development of third-party plugins." - ) - rich.print("-> Data should be backed up before performing this upgrade. The 'prevent_deploy' flag has been set in your config file and must be manually removed to deploy." - ) - + ) + rich.print( + "-> Data should be backed up before performing this upgrade. The 'prevent_deploy' flag has been set in your config file and must be manually removed to deploy." + ) + # Setting the following flag will prevent deployment and display guidance to the user # which they can override if they are happy they understand the situation. config["prevent_deploy"] = True - - + # Nebari version 2023.9.1 upgrades JupyterHub to 3.1. CDS Dashboards are only compatible with # JupyterHub versions 1.X and so will be removed during upgrade. rich.print("\n ⚠️ Deprecation Warning ⚠️") @@ -533,27 +539,27 @@ def _version_specific_upgrade( f"-> CDS dashboards are no longer supported in Nebari version [green]{self.version}[/green] and will be uninstalled." ) if config.get("cdsdashboards"): - rich.print( - f"-> Removing cdsdashboards from config file." - ) + rich.print("-> Removing cdsdashboards from config file.") del config["cdsdashboards"] # Kubernetes version check. Minimum Kubernetes version is 1.26 # JupyterHub Helm chart 2.0.0 (app version 3.0.0) requires K8S Version >=1.23. (reference: https://z2jh.jupyter.org/en/stable/) - + # Upgrade instructions for various providers upgrade_messages = { "aws": "Please upgrade your EKS cluster outside of Nebari following Amazon Web Services guide: https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html.", "azure": "Please upgrade your AKS cluster outside of Nebari following Azure Cloud's guide: https://learn.microsoft.com/en-us/azure/aks/upgrade-cluster.", "gcp": "Please upgrade your GKE cluster outside of Nebari following Google Cloud Platform's guide: https://cloud.google.com/kubernetes-engine/docs/how-to/upgrading-a-cluster.", - "do": "Please upgrade your DOKS cluster outside of Nebari following Digital Ocean's guide: https://docs.digitalocean.com/products/kubernetes/how-to/upgrade-cluster/" + "do": "Please upgrade your DOKS cluster outside of Nebari following Digital Ocean's guide: https://docs.digitalocean.com/products/kubernetes/how-to/upgrade-cluster/", } provider = config["provider"] provider_config_block = get_provider_config_block_name(provider) # Get current Kubernetes version if available in config. - current_version = config.get(provider_config_block, {}).get("kubernetes_version", "NA") + current_version = config.get(provider_config_block, {}).get( + "kubernetes_version", "NA" + ) # Convert to decimal prefix if provider is Digital Ocean if provider == "do": @@ -564,42 +570,39 @@ def _version_specific_upgrade( if not current_version == "NA": try: current_version = float(current_version) - except: - current_version = "NA" + except ValueError: + current_version = "NA" # Handle checks for when Kubernetes version should be detectable - if provider in ["aws","azure","gcp","do"]: - + if provider in ["aws", "azure", "gcp", "do"]: # Kubernetes version not found in provider block if current_version == "NA": rich.print("\n ⚠️ Warning ⚠️") rich.print( - f"-> Unable to detect Kubernetes version for provider {provider}. Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26 or greater. Please confirm your Kubernetes version is configured before upgrading." + f"-> Unable to detect Kubernetes version for provider {provider}. Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26 or greater. Please confirm your Kubernetes version is configured before upgrading." ) - exit(1) + exit(1) # Kubernetes version less than required minimum if current_version < 1.26: - rich.print("\n ⚠️ Warning ⚠️") - rich.print( - f"-> Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26 or greater. Your configured Kubernetes version is [red]{current_version}[/red]." - ) - rich.print( - upgrade_messages[provider] - ) - rich.print( - f"-> Once you have completed the upgrade, update the value of {provider_config_block}.kubernetes_version in your config file to match and run the upgrade command again." - ) - exit(1) + rich.print("\n ⚠️ Warning ⚠️") + rich.print( + f"-> Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26 or greater. Your configured Kubernetes version is [red]{current_version}[/red]." + ) + rich.print(upgrade_messages[provider]) + rich.print( + f"-> Once you have completed the upgrade, update the value of {provider_config_block}.kubernetes_version in your config file to match and run the upgrade command again." + ) + exit(1) else: rich.print("\n ⚠️ Warning ⚠️") rich.print( f"-> Unable to detect Kubernetes version for provider {provider}. Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26 or greater." - ) + ) rich.print( - f"-> Please ensure your Kubernetes version is up-to-date before proceeding." - ) + "-> Please ensure your Kubernetes version is up-to-date before proceeding." + ) return config diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index 977a3084d0..36e38d4e59 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -315,14 +315,15 @@ def construct_azure_resource_group_name( return f"{base_resource_group_name}{suffix}" return f"{project_name}-{namespace}{suffix}" + def get_do_k8s_version_prefix(do_k8s_version): # Split the input string by the first decimal point - parts = do_k8s_version.split('.', 1) + parts = do_k8s_version.split(".", 1) if len(parts) == 2: # Extract the part before the second decimal point - before_second_decimal = parts[0] + '.' + parts[1].split('.')[0] - + before_second_decimal = parts[0] + "." + parts[1].split(".")[0] + try: # Convert the extracted part to a float result = float(before_second_decimal) @@ -334,16 +335,16 @@ def get_do_k8s_version_prefix(do_k8s_version): # Handle the case where there is no second decimal point return None -def get_provider_config_block_name(provider): +def get_provider_config_block_name(provider): PROVIDER_CONFIG_NAMES = { "aws": "amazon_web_services", "azure": "azure_cloud", "do": "digital_ocean", - "gcp": "google_cloud" + "gcp": "google_cloud", } if provider in PROVIDER_CONFIG_NAMES.keys(): return PROVIDER_CONFIG_NAMES[provider] else: - return provider \ No newline at end of file + return provider diff --git a/tests/tests_unit/test_cli_upgrade.py b/tests/tests_unit/test_cli_upgrade.py index 4ce778a21a..404cd5f3b3 100644 --- a/tests/tests_unit/test_cli_upgrade.py +++ b/tests/tests_unit/test_cli_upgrade.py @@ -8,10 +8,9 @@ import _nebari.upgrade import _nebari.version -from _nebari.utils import get_provider_config_block_name from _nebari.cli import create_cli from _nebari.constants import AZURE_DEFAULT_REGION - +from _nebari.utils import get_provider_config_block_name MOCK_KUBERNETES_VERSIONS = { "aws": ["1.20"], @@ -26,6 +25,7 @@ "do": ["nyc3"], } + # can't upgrade to a previous version that doesn't have a corresponding # UpgradeStep derived class. without these dummy classes, the rendered # nebari-config.yaml will have the wrong version @@ -397,12 +397,13 @@ def test_cli_upgrade_to_0_4_0_fails_for_custom_auth_without_attempt_fixes(): with open(tmp_file.resolve(), "r") as c: assert yaml.safe_load(c) == nebari_config + def test_cli_upgrade_to_2023_9_1_cdsdashboard_removed(monkeypatch: pytest.MonkeyPatch): start_version = "2023.7.2" end_version = "2023.9.1" addl_config = yaml.safe_load( - f""" + """ cdsdashboards: enabled: true cds_hide_user_named_servers: true @@ -415,31 +416,33 @@ def test_cli_upgrade_to_2023_9_1_cdsdashboard_removed(monkeypatch: pytest.Monkey start_version, end_version, addl_args=["--attempt-fixes"], - addl_config=addl_config + addl_config=addl_config, ) - + assert not upgraded.get("cdsdashboards") assert upgraded.get("prevent_deploy") + @pytest.mark.parametrize( - ("provider","k8s_status"), + ("provider", "k8s_status"), [ - ("aws", "compatible"), - ("aws", "incompatible"), - ("aws", "invalid"), - ("azure", "compatible"), - ("azure", "incompatible"), - ("azure", "invalid"), - ("do", "compatible"), - ("do", "incompatible"), - ("do", "invalid"), - ("gcp", "compatible"), - ("gcp", "incompatible"), - ("gcp", "invalid"), - ] + ("aws", "compatible"), + ("aws", "incompatible"), + ("aws", "invalid"), + ("azure", "compatible"), + ("azure", "incompatible"), + ("azure", "invalid"), + ("do", "compatible"), + ("do", "incompatible"), + ("do", "invalid"), + ("gcp", "compatible"), + ("gcp", "incompatible"), + ("gcp", "invalid"), + ], ) - -def test_cli_upgrade_to_2023_9_1_kubernetes_validations(monkeypatch: pytest.MonkeyPatch, provider: str, k8s_status: str): +def test_cli_upgrade_to_2023_9_1_kubernetes_validations( + monkeypatch: pytest.MonkeyPatch, provider: str, k8s_status: str +): start_version = "2023.7.2" end_version = "2023.9.1" monkeypatch.setattr(_nebari.upgrade, "__version__", end_version) @@ -447,7 +450,11 @@ def test_cli_upgrade_to_2023_9_1_kubernetes_validations(monkeypatch: pytest.Monk kubernetes_configs = { "aws": {"incompatible": "1.19", "compatible": "1.26", "invalid": "badname"}, "azure": {"incompatible": "1.23", "compatible": "1.26", "invalid": "badname"}, - "do": {"incompatible": "1.19.2-do.3", "compatible": "1.26.0-do.custom", "invalid": "badname"}, + "do": { + "incompatible": "1.19.2-do.3", + "compatible": "1.26.0-do.custom", + "invalid": "badname", + }, "gcp": {"incompatible": "1.23", "compatible": "1.26", "invalid": "badname"}, } @@ -456,7 +463,7 @@ def test_cli_upgrade_to_2023_9_1_kubernetes_validations(monkeypatch: pytest.Monk "aws": "Please upgrade your EKS cluster outside of Nebari following Amazon Web Services guide", "azure": "Please upgrade your AKS cluster outside of Nebari following Azure Cloud's guide", "gcp": "Please upgrade your GKE cluster outside of Nebari following Google Cloud Platform's guide", - "do": "Please upgrade your DOKS cluster outside of Nebari following Digital Ocean's guide" + "do": "Please upgrade your DOKS cluster outside of Nebari following Digital Ocean's guide", } with tempfile.TemporaryDirectory() as tmp: @@ -464,7 +471,7 @@ def test_cli_upgrade_to_2023_9_1_kubernetes_validations(monkeypatch: pytest.Monk assert tmp_file.exists() is False nebari_config = yaml.safe_load( - f""" + f""" project_name: test provider: {provider} domain: test.example.com @@ -478,19 +485,19 @@ def test_cli_upgrade_to_2023_9_1_kubernetes_validations(monkeypatch: pytest.Monk region: {MOCK_CLOUD_REGIONS.get(provider, {})[0]} kubernetes_version: {kubernetes_configs[provider][k8s_status]} """ - ) + ) with open(tmp_file.resolve(), "w") as f: - yaml.dump(nebari_config, f) - + yaml.dump(nebari_config, f) + assert tmp_file.exists() is True app = create_cli() result = runner.invoke(app, ["upgrade", "--config", tmp_file.resolve()]) - + if k8s_status == "incompatible": assert 1 == result.exit_code assert result.exception - assert upgrade_messages[provider] in result.stdout.replace("\n","") + assert upgrade_messages[provider] in result.stdout.replace("\n", "") if k8s_status == "compatible": assert 0 == result.exit_code @@ -505,7 +512,13 @@ def test_cli_upgrade_to_2023_9_1_kubernetes_validations(monkeypatch: pytest.Monk if k8s_status == "invalid": assert 1 == result.exit_code assert result.exception - assert "Unable to detect Kubernetes version for provider {}".format(provider) in result.stdout + assert ( + "Unable to detect Kubernetes version for provider {}".format( + provider + ) + in result.stdout + ) + def assert_nebari_upgrade_success( monkeypatch: pytest.MonkeyPatch, From 0e624e8e5a135c353d51bedb6321d65dc247ef45 Mon Sep 17 00:00:00 2001 From: iameskild Date: Fri, 22 Sep 2023 23:07:53 -0700 Subject: [PATCH 06/17] Ensure GCP k8s is upgraded properly --- .../template/gcp/modules/kubernetes/main.tf | 4 ---- .../infrastructure/template/gcp/versions.tf | 2 +- src/_nebari/upgrade.py | 19 ++++++++++++------- src/_nebari/utils.py | 6 ++++-- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/main.tf index f16bae35e2..c4b18f32ad 100644 --- a/src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/main.tf +++ b/src/_nebari/stages/infrastructure/template/gcp/modules/kubernetes/main.tf @@ -57,10 +57,6 @@ resource "google_container_cluster" "main" { } } - cost_management_config { - enabled = true - } - lifecycle { ignore_changes = [ node_locations diff --git a/src/_nebari/stages/infrastructure/template/gcp/versions.tf b/src/_nebari/stages/infrastructure/template/gcp/versions.tf index 9641dfac90..ddea3c185c 100644 --- a/src/_nebari/stages/infrastructure/template/gcp/versions.tf +++ b/src/_nebari/stages/infrastructure/template/gcp/versions.tf @@ -2,7 +2,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = "4.72.0" + version = "4.8.0" } } required_version = ">= 1.0" diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 499be6cb1e..b4446d7ac1 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -12,7 +12,7 @@ from _nebari.config import backup_configuration from _nebari.utils import ( - get_do_k8s_version_prefix, + get_k8s_version_prefix, get_provider_config_block_name, load_yaml, yaml, @@ -527,6 +527,9 @@ def _version_specific_upgrade( rich.print( "-> Data should be backed up before performing this upgrade. The 'prevent_deploy' flag has been set in your config file and must be manually removed to deploy." ) + rich.print( + "-> Please also run the [green]rm -rf stages[/green] so that we can regenerate an updated set of Terraform scripts for your deployment." + ) # Setting the following flag will prevent deployment and display guidance to the user # which they can override if they are happy they understand the situation. @@ -562,9 +565,8 @@ def _version_specific_upgrade( ) # Convert to decimal prefix if provider is Digital Ocean - if provider == "do": - current_version = get_do_k8s_version_prefix(current_version) - rich.print(current_version) + if provider == "do" or provider == "gcp": + current_version = get_k8s_version_prefix(current_version) # Try to convert known Kubernetes versions to float. if not current_version == "NA": @@ -581,7 +583,6 @@ def _version_specific_upgrade( rich.print( f"-> Unable to detect Kubernetes version for provider {provider}. Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26 or greater. Please confirm your Kubernetes version is configured before upgrading." ) - exit(1) # Kubernetes version less than required minimum if current_version < 1.26: @@ -590,10 +591,14 @@ def _version_specific_upgrade( f"-> Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26 or greater. Your configured Kubernetes version is [red]{current_version}[/red]." ) rich.print(upgrade_messages[provider]) + version_diff = round(1.26 - current_version, 2) + if version_diff > 0.01: + rich.print( + "-> The Kubernetes version is multiple minor versions behind the minimum required version. You will need to perform the upgrade one minor version at a time. For example, if your current version is 1.23, you will need to upgrade to 1.24, then 1.25, and finally 1.26." + ) rich.print( - f"-> Once you have completed the upgrade, update the value of {provider_config_block}.kubernetes_version in your config file to match and run the upgrade command again." + f"-> Once you have completed the upgrade, update the value of [green]{provider_config_block}.kubernetes_version[/green] in your config file to match and run the upgrade command again." ) - exit(1) else: rich.print("\n ⚠️ Warning ⚠️") diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index 36e38d4e59..9f1ced5c1c 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -316,7 +316,9 @@ def construct_azure_resource_group_name( return f"{project_name}-{namespace}{suffix}" -def get_do_k8s_version_prefix(do_k8s_version): +def get_k8s_version_prefix(do_k8s_version: str) -> str: + """Return the major.minor version of the k8s version string.""" + # Split the input string by the first decimal point parts = do_k8s_version.split(".", 1) @@ -341,7 +343,7 @@ def get_provider_config_block_name(provider): "aws": "amazon_web_services", "azure": "azure_cloud", "do": "digital_ocean", - "gcp": "google_cloud", + "gcp": "google_cloud_platform", } if provider in PROVIDER_CONFIG_NAMES.keys(): From 008d1176bb94823171d36170056382ef1669345e Mon Sep 17 00:00:00 2001 From: iameskild Date: Sun, 24 Sep 2023 10:50:20 -0700 Subject: [PATCH 07/17] Update upgrade command, tests --- src/_nebari/upgrade.py | 8 +++++--- src/_nebari/utils.py | 3 +-- tests/tests_unit/test_cli_validate.py | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index b4446d7ac1..125a2b9a61 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -564,9 +564,11 @@ def _version_specific_upgrade( "kubernetes_version", "NA" ) - # Convert to decimal prefix if provider is Digital Ocean - if provider == "do" or provider == "gcp": + # Convert to decimal prefix + if provider in ["do", "gcp", "azure"]: current_version = get_k8s_version_prefix(current_version) + elif provider == "aws": + current_version = float(current_version) # Try to convert known Kubernetes versions to float. if not current_version == "NA": @@ -588,7 +590,7 @@ def _version_specific_upgrade( if current_version < 1.26: rich.print("\n ⚠️ Warning ⚠️") rich.print( - f"-> Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26 or greater. Your configured Kubernetes version is [red]{current_version}[/red]." + f"-> Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26. Your configured Kubernetes version is [red]{current_version}[/red]." ) rich.print(upgrade_messages[provider]) version_diff = round(1.26 - current_version, 2) diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index 9f1ced5c1c..e3bbb804bc 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -325,7 +325,6 @@ def get_k8s_version_prefix(do_k8s_version: str) -> str: if len(parts) == 2: # Extract the part before the second decimal point before_second_decimal = parts[0] + "." + parts[1].split(".")[0] - try: # Convert the extracted part to a float result = float(before_second_decimal) @@ -341,7 +340,7 @@ def get_k8s_version_prefix(do_k8s_version: str) -> str: def get_provider_config_block_name(provider): PROVIDER_CONFIG_NAMES = { "aws": "amazon_web_services", - "azure": "azure_cloud", + "azure": "azure", "do": "digital_ocean", "gcp": "google_cloud_platform", } diff --git a/tests/tests_unit/test_cli_validate.py b/tests/tests_unit/test_cli_validate.py index 13955c1fc7..0894da9273 100644 --- a/tests/tests_unit/test_cli_validate.py +++ b/tests/tests_unit/test_cli_validate.py @@ -7,6 +7,7 @@ import yaml from typer.testing import CliRunner +from _nebari._version import __version__ from _nebari.cli import create_cli TEST_DATA_DIR = Path(__file__).resolve().parent / "cli_validate" @@ -14,6 +15,17 @@ runner = CliRunner() +def _update_yaml_file(file_path: Path, key: str, value: Any): + """Utility function to update a yaml file with a new key/value pair.""" + with open(file_path, "r") as f: + yaml_data = yaml.safe_load(f) + + yaml_data[key] = value + + with open(file_path, "w") as f: + yaml.safe_dump(yaml_data, f) + + @pytest.mark.parametrize( "args, exit_code, content", [ @@ -62,6 +74,9 @@ def test_cli_validate_local_happy_path(config_yaml: str): test_file = TEST_DATA_DIR / config_yaml assert test_file.exists() is True + # update the test file with the current version + _update_yaml_file(test_file, "nebari_version", __version__) + app = create_cli() result = runner.invoke(app, ["validate", "--config", test_file]) assert not result.exception From 8c515e22b29e5644676097bc5341c270e1eb3919 Mon Sep 17 00:00:00 2001 From: iameskild Date: Sun, 24 Sep 2023 11:50:54 -0700 Subject: [PATCH 08/17] Clean up tests, only run if nebari version is available --- src/_nebari/upgrade.py | 22 +++++++--------------- src/_nebari/utils.py | 5 +++-- tests/tests_unit/test_cli_upgrade.py | 27 +++++++++++++-------------- 3 files changed, 23 insertions(+), 31 deletions(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 125a2b9a61..0c02eb65d8 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -27,6 +27,8 @@ ) ARGO_JUPYTER_SCHEDULER_REPO = "https://github.com/nebari-dev/argo-jupyter-scheduler" +UPGRADE_KUBERNETES_MESSAGE = "Please see the [green][link=https://www.nebari.dev/docs/how-tos/kubernetes-version-upgrade]Kubernetes upgrade docs[/link][/green] for more information." + def do_upgrade(config_filename, attempt_fixes=False): config = load_yaml(config_filename) @@ -548,14 +550,6 @@ def _version_specific_upgrade( # Kubernetes version check. Minimum Kubernetes version is 1.26 # JupyterHub Helm chart 2.0.0 (app version 3.0.0) requires K8S Version >=1.23. (reference: https://z2jh.jupyter.org/en/stable/) - # Upgrade instructions for various providers - upgrade_messages = { - "aws": "Please upgrade your EKS cluster outside of Nebari following Amazon Web Services guide: https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html.", - "azure": "Please upgrade your AKS cluster outside of Nebari following Azure Cloud's guide: https://learn.microsoft.com/en-us/azure/aks/upgrade-cluster.", - "gcp": "Please upgrade your GKE cluster outside of Nebari following Google Cloud Platform's guide: https://cloud.google.com/kubernetes-engine/docs/how-to/upgrading-a-cluster.", - "do": "Please upgrade your DOKS cluster outside of Nebari following Digital Ocean's guide: https://docs.digitalocean.com/products/kubernetes/how-to/upgrade-cluster/", - } - provider = config["provider"] provider_config_block = get_provider_config_block_name(provider) @@ -565,16 +559,14 @@ def _version_specific_upgrade( ) # Convert to decimal prefix - if provider in ["do", "gcp", "azure"]: + if provider in ["aws", "azure", "gcp", "do"]: current_version = get_k8s_version_prefix(current_version) - elif provider == "aws": - current_version = float(current_version) # Try to convert known Kubernetes versions to float. if not current_version == "NA": try: current_version = float(current_version) - except ValueError: + except TypeError: current_version = "NA" # Handle checks for when Kubernetes version should be detectable @@ -583,16 +575,16 @@ def _version_specific_upgrade( if current_version == "NA": rich.print("\n ⚠️ Warning ⚠️") rich.print( - f"-> Unable to detect Kubernetes version for provider {provider}. Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26 or greater. Please confirm your Kubernetes version is configured before upgrading." + f"-> Unable to detect Kubernetes version for provider {provider}. Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26. Please confirm your Kubernetes version is configured before upgrading." ) # Kubernetes version less than required minimum - if current_version < 1.26: + if isinstance(current_version, float) and current_version < 1.26: rich.print("\n ⚠️ Warning ⚠️") rich.print( f"-> Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26. Your configured Kubernetes version is [red]{current_version}[/red]." ) - rich.print(upgrade_messages[provider]) + rich.print(UPGRADE_KUBERNETES_MESSAGE) version_diff = round(1.26 - current_version, 2) if version_diff > 0.01: rich.print( diff --git a/src/_nebari/utils.py b/src/_nebari/utils.py index e3bbb804bc..3378116a1d 100644 --- a/src/_nebari/utils.py +++ b/src/_nebari/utils.py @@ -316,11 +316,12 @@ def construct_azure_resource_group_name( return f"{project_name}-{namespace}{suffix}" -def get_k8s_version_prefix(do_k8s_version: str) -> str: +def get_k8s_version_prefix(k8s_version: str) -> str: """Return the major.minor version of the k8s version string.""" + k8s_version = str(k8s_version) # Split the input string by the first decimal point - parts = do_k8s_version.split(".", 1) + parts = k8s_version.split(".", 1) if len(parts) == 2: # Extract the part before the second decimal point diff --git a/tests/tests_unit/test_cli_upgrade.py b/tests/tests_unit/test_cli_upgrade.py index 404cd5f3b3..e2650c603d 100644 --- a/tests/tests_unit/test_cli_upgrade.py +++ b/tests/tests_unit/test_cli_upgrade.py @@ -1,3 +1,4 @@ +import re import tempfile from pathlib import Path from typing import Any, Dict, List @@ -10,6 +11,7 @@ import _nebari.version from _nebari.cli import create_cli from _nebari.constants import AZURE_DEFAULT_REGION +from _nebari.upgrade import UPGRADE_KUBERNETES_MESSAGE from _nebari.utils import get_provider_config_block_name MOCK_KUBERNETES_VERSIONS = { @@ -423,6 +425,10 @@ def test_cli_upgrade_to_2023_9_1_cdsdashboard_removed(monkeypatch: pytest.Monkey assert upgraded.get("prevent_deploy") +@pytest.mark.skipif( + _nebari.upgrade.__version__ < "2023.9.1", + reason="This test is only valid for versions <= 2023.9.1", +) @pytest.mark.parametrize( ("provider", "k8s_status"), [ @@ -458,14 +464,6 @@ def test_cli_upgrade_to_2023_9_1_kubernetes_validations( "gcp": {"incompatible": "1.23", "compatible": "1.26", "invalid": "badname"}, } - # Incompatible provider tests - upgrade_messages = { - "aws": "Please upgrade your EKS cluster outside of Nebari following Amazon Web Services guide", - "azure": "Please upgrade your AKS cluster outside of Nebari following Azure Cloud's guide", - "gcp": "Please upgrade your GKE cluster outside of Nebari following Google Cloud Platform's guide", - "do": "Please upgrade your DOKS cluster outside of Nebari following Digital Ocean's guide", - } - with tempfile.TemporaryDirectory() as tmp: tmp_file = Path(tmp).resolve() / "nebari-config.yaml" assert tmp_file.exists() is False @@ -495,12 +493,15 @@ def test_cli_upgrade_to_2023_9_1_kubernetes_validations( result = runner.invoke(app, ["upgrade", "--config", tmp_file.resolve()]) if k8s_status == "incompatible": - assert 1 == result.exit_code - assert result.exception - assert upgrade_messages[provider] in result.stdout.replace("\n", "") + UPGRADE_KUBERNETES_MESSAGE_WO_BRACKETS = re.sub( + r"\[.*?\]", "", UPGRADE_KUBERNETES_MESSAGE + ) + assert UPGRADE_KUBERNETES_MESSAGE_WO_BRACKETS in result.stdout.replace( + "\n", "" + ) if k8s_status == "compatible": - assert 0 == result.exit_code + # assert 0 == result.exit_code assert not result.exception assert "Saving new config file" in result.stdout @@ -510,8 +511,6 @@ def test_cli_upgrade_to_2023_9_1_kubernetes_validations( assert end_version == upgraded["nebari_version"] if k8s_status == "invalid": - assert 1 == result.exit_code - assert result.exception assert ( "Unable to detect Kubernetes version for provider {}".format( provider From d046d58bfa4adba22bb56ecc45adfe5f2fd66a1f Mon Sep 17 00:00:00 2001 From: iameskild Date: Sun, 24 Sep 2023 11:55:15 -0700 Subject: [PATCH 09/17] Clean up docs message --- src/_nebari/upgrade.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 0c02eb65d8..7f62ffbe40 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -582,9 +582,8 @@ def _version_specific_upgrade( if isinstance(current_version, float) and current_version < 1.26: rich.print("\n ⚠️ Warning ⚠️") rich.print( - f"-> Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26. Your configured Kubernetes version is [red]{current_version}[/red]." + f"-> Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26. Your configured Kubernetes version is [red]{current_version}[/red]. {UPGRADE_KUBERNETES_MESSAGE}" ) - rich.print(UPGRADE_KUBERNETES_MESSAGE) version_diff = round(1.26 - current_version, 2) if version_diff > 0.01: rich.print( From d7df1bb8b5b587f37487358c226eeea41babf812 Mon Sep 17 00:00:00 2001 From: iameskild Date: Sun, 24 Sep 2023 17:30:46 -0700 Subject: [PATCH 10/17] Update message --- src/_nebari/upgrade.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 7f62ffbe40..acdd9db849 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -587,10 +587,10 @@ def _version_specific_upgrade( version_diff = round(1.26 - current_version, 2) if version_diff > 0.01: rich.print( - "-> The Kubernetes version is multiple minor versions behind the minimum required version. You will need to perform the upgrade one minor version at a time. For example, if your current version is 1.23, you will need to upgrade to 1.24, then 1.25, and finally 1.26." + "-> The Kubernetes version is multiple minor versions behind the minimum required version. You will need to perform the upgrade one minor version at a time. For example, if your current version is 1.24, you will need to upgrade to 1.25, and then 1.26." ) rich.print( - f"-> Once you have completed the upgrade, update the value of [green]{provider_config_block}.kubernetes_version[/green] in your config file to match and run the upgrade command again." + f"-> Update the value of [green]{provider_config_block}.kubernetes_version[/green] in your config file to a newer version of Kubernetes and redeploy." ) else: From 33198a7f8360246d0f62ca3f4fb4681619128829 Mon Sep 17 00:00:00 2001 From: iameskild Date: Sun, 24 Sep 2023 17:36:36 -0700 Subject: [PATCH 11/17] Uncomment assert --- tests/tests_unit/test_cli_upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tests_unit/test_cli_upgrade.py b/tests/tests_unit/test_cli_upgrade.py index e2650c603d..4863fbff43 100644 --- a/tests/tests_unit/test_cli_upgrade.py +++ b/tests/tests_unit/test_cli_upgrade.py @@ -501,7 +501,7 @@ def test_cli_upgrade_to_2023_9_1_kubernetes_validations( ) if k8s_status == "compatible": - # assert 0 == result.exit_code + assert 0 == result.exit_code assert not result.exception assert "Saving new config file" in result.stdout From a7762aacecee860893f409391e8b940293704f06 Mon Sep 17 00:00:00 2001 From: iameskild Date: Mon, 25 Sep 2023 08:09:01 -0700 Subject: [PATCH 12/17] Add skipif to test_cli_upgrade_to_2023_9_1_cdsdashboard_removed --- tests/tests_unit/test_cli_upgrade.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/tests_unit/test_cli_upgrade.py b/tests/tests_unit/test_cli_upgrade.py index 4863fbff43..45a98d2b35 100644 --- a/tests/tests_unit/test_cli_upgrade.py +++ b/tests/tests_unit/test_cli_upgrade.py @@ -400,6 +400,10 @@ def test_cli_upgrade_to_0_4_0_fails_for_custom_auth_without_attempt_fixes(): assert yaml.safe_load(c) == nebari_config +@pytest.mark.skipif( + _nebari.upgrade.__version__ < "2023.9.1", + reason="This test is only valid for versions <= 2023.9.1", +) def test_cli_upgrade_to_2023_9_1_cdsdashboard_removed(monkeypatch: pytest.MonkeyPatch): start_version = "2023.7.2" end_version = "2023.9.1" From aae93a9af6826d879ee1f4c36d7e674e76738295 Mon Sep 17 00:00:00 2001 From: iameskild Date: Mon, 25 Sep 2023 17:25:50 -0700 Subject: [PATCH 13/17] Add DANGER warning for AWS, clean up --- src/_nebari/config.py | 1 - src/_nebari/constants.py | 2 +- src/_nebari/upgrade.py | 23 +++++++++++++---------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/_nebari/config.py b/src/_nebari/config.py index 00c9679368..d1b2f42944 100644 --- a/src/_nebari/config.py +++ b/src/_nebari/config.py @@ -115,4 +115,3 @@ def backup_configuration(filename: pathlib.Path, extrasuffix: str = ""): i = i + 1 filename.rename(backup_filename) - print(f"Backing up {filename} as {backup_filename}") diff --git a/src/_nebari/constants.py b/src/_nebari/constants.py index 3c51f37701..48e7ec0aff 100644 --- a/src/_nebari/constants.py +++ b/src/_nebari/constants.py @@ -8,7 +8,7 @@ # 04-kubernetes-ingress DEFAULT_TRAEFIK_IMAGE_TAG = "2.9.1" -HIGHEST_SUPPORTED_K8S_VERSION = ("1", "26", "7") +HIGHEST_SUPPORTED_K8S_VERSION = ("1", "26", "9") DEFAULT_GKE_RELEASE_CHANNEL = "UNSPECIFIED" DEFAULT_NEBARI_DASK_VERSION = CURRENT_RELEASE diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index acdd9db849..bb2b8ccbae 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -55,7 +55,6 @@ def do_upgrade(config_filename, attempt_fixes=False): raise e start_version = config.get("nebari_version", "") - print("start_version: ", start_version) UpgradeStep.upgrade( config, start_version, __version__, config_filename, attempt_fixes @@ -105,7 +104,6 @@ def upgrade( """ starting_ver = rounded_ver_parse(start_version or "0.0.0") finish_ver = rounded_ver_parse(finish_version) - print("finish_ver: ", finish_ver) if finish_ver < starting_ver: raise ValueError( @@ -123,8 +121,6 @@ def upgrade( key=rounded_ver_parse, ) - print("step_versions: ", step_versions) - current_start_version = start_version for stepcls in [cls._steps[str(v)] for v in step_versions]: step = stepcls() @@ -522,12 +518,12 @@ def _version_specific_upgrade( ): # Upgrading to 2023.9.1 is considered high-risk because it includes a major refacto # to introduce the extension mechanism system. - rich.print("\n ⚠️ Warning ⚠️") + rich.print("\n ⚠️ Warning ⚠️") rich.print( f"-> Nebari version [green]{self.version}[/green] includes a major refactor to introduce an extension mechanism that supports the development of third-party plugins." ) rich.print( - "-> Data should be backed up before performing this upgrade. The 'prevent_deploy' flag has been set in your config file and must be manually removed to deploy." + "-> Data should be backed up before performing this upgrade ([green][link=https://www.nebari.dev/docs/how-tos/manual-backup]see docs[/link][/green]) The 'prevent_deploy' flag has been set in your config file and must be manually removed to deploy." ) rich.print( "-> Please also run the [green]rm -rf stages[/green] so that we can regenerate an updated set of Terraform scripts for your deployment." @@ -539,7 +535,7 @@ def _version_specific_upgrade( # Nebari version 2023.9.1 upgrades JupyterHub to 3.1. CDS Dashboards are only compatible with # JupyterHub versions 1.X and so will be removed during upgrade. - rich.print("\n ⚠️ Deprecation Warning ⚠️") + rich.print("\n ⚠️ Deprecation Warning ⚠️") rich.print( f"-> CDS dashboards are no longer supported in Nebari version [green]{self.version}[/green] and will be uninstalled." ) @@ -573,14 +569,14 @@ def _version_specific_upgrade( if provider in ["aws", "azure", "gcp", "do"]: # Kubernetes version not found in provider block if current_version == "NA": - rich.print("\n ⚠️ Warning ⚠️") + rich.print("\n ⚠️ Warning ⚠️") rich.print( f"-> Unable to detect Kubernetes version for provider {provider}. Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26. Please confirm your Kubernetes version is configured before upgrading." ) # Kubernetes version less than required minimum if isinstance(current_version, float) and current_version < 1.26: - rich.print("\n ⚠️ Warning ⚠️") + rich.print("\n ⚠️ Warning ⚠️") rich.print( f"-> Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26. Your configured Kubernetes version is [red]{current_version}[/red]. {UPGRADE_KUBERNETES_MESSAGE}" ) @@ -594,13 +590,20 @@ def _version_specific_upgrade( ) else: - rich.print("\n ⚠️ Warning ⚠️") + rich.print("\n ⚠️ Warning ⚠️") rich.print( f"-> Unable to detect Kubernetes version for provider {provider}. Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26 or greater." ) rich.print( "-> Please ensure your Kubernetes version is up-to-date before proceeding." ) + + if provider == "aws": + rich.print("\n ⚠️ DANGER ⚠️") + rich.print( + "-> This version upgrade will result in your cluster being completely torn down and redeployed. Please ensure you have backed up any data you wish to keep before proceeding!!!" + ) + return config From 89ede8f39574199ac04126e9107e2aeba2266ee6 Mon Sep 17 00:00:00 2001 From: iameskild Date: Mon, 25 Sep 2023 19:34:58 -0700 Subject: [PATCH 14/17] Add warning for AWS to 2023.7.2 upgrade --- src/_nebari/upgrade.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index bb2b8ccbae..0d214c9f83 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -494,6 +494,7 @@ def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): argo = config.get("argo_workflows", {}) + provider = config["provider"] if argo.get("enabled"): response = Prompt.ask( f"\nDo you want to enable the [green][link={NEBARI_WORKFLOW_CONTROLLER_DOCS}]Nebari Workflow Controller[/link][/green], required for [green][link={ARGO_JUPYTER_SCHEDULER_REPO}]Argo-Jupyter-Scheduler[/link][green]? [Y/n] ", @@ -507,6 +508,12 @@ def _version_specific_upgrade( f"-> [green]{self.version}[/green] is the last Nebari version that supports CDS Dashboards" ) + if provider == "aws": + rich.print("\n ⚠️ DANGER ⚠️") + rich.print( + "-> This version upgrade will result in your cluster being completely torn down and redeployed. Please ensure you have backed up any data you wish to keep before proceeding!!!" + ) + return config From 3c333d232a27123e4fe6d9555b97337ca911b574 Mon Sep 17 00:00:00 2001 From: iameskild Date: Mon, 25 Sep 2023 20:01:48 -0700 Subject: [PATCH 15/17] Add message, prevent_deploy, tests for 2023.7.1 --- src/_nebari/upgrade.py | 25 ++++++++++++++++++------- tests/tests_unit/test_cli_upgrade.py | 26 ++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index 0d214c9f83..e67c7082cc 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -487,6 +487,24 @@ def _version_specific_upgrade( return config +class Upgrade_2023_7_1(UpgradeStep): + version = "2023.7.1" + + def _version_specific_upgrade( + self, config, start_version, config_filename: Path, *args, **kwargs + ): + provider = config["provider"] + if provider == "aws": + rich.print("\n ⚠️ DANGER ⚠️") + rich.print( + "-> This version upgrade will result in your cluster being completely torn down and redeployed. Please ensure you have backed up any data you wish to keep before proceeding!!!", + "The 'prevent_deploy' flag has been set in your config file and must be manually removed to deploy.", + ) + config["prevent_deploy"] = True + + return config + + class Upgrade_2023_7_2(UpgradeStep): version = "2023.7.2" @@ -494,7 +512,6 @@ def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): argo = config.get("argo_workflows", {}) - provider = config["provider"] if argo.get("enabled"): response = Prompt.ask( f"\nDo you want to enable the [green][link={NEBARI_WORKFLOW_CONTROLLER_DOCS}]Nebari Workflow Controller[/link][/green], required for [green][link={ARGO_JUPYTER_SCHEDULER_REPO}]Argo-Jupyter-Scheduler[/link][green]? [Y/n] ", @@ -508,12 +525,6 @@ def _version_specific_upgrade( f"-> [green]{self.version}[/green] is the last Nebari version that supports CDS Dashboards" ) - if provider == "aws": - rich.print("\n ⚠️ DANGER ⚠️") - rich.print( - "-> This version upgrade will result in your cluster being completely torn down and redeployed. Please ensure you have backed up any data you wish to keep before proceeding!!!" - ) - return config diff --git a/tests/tests_unit/test_cli_upgrade.py b/tests/tests_unit/test_cli_upgrade.py index 45a98d2b35..e0fe685aff 100644 --- a/tests/tests_unit/test_cli_upgrade.py +++ b/tests/tests_unit/test_cli_upgrade.py @@ -104,11 +104,28 @@ def test_cli_upgrade_2023_4_1_to_2023_5_1(monkeypatch: pytest.MonkeyPatch): ) +@pytest.mark.parametrize( + "provider", + ["aws", "azure", "do", "gcp"], +) +def test_cli_upgrade_2023_5_1_to_2023_7_1( + monkeypatch: pytest.MonkeyPatch, provider: str +): + config = assert_nebari_upgrade_success( + monkeypatch, "2023.5.1", "2023.7.1", provider=provider + ) + prevent_deploy = config.get("prevent_deploy") + if provider == "aws": + assert prevent_deploy + else: + assert not prevent_deploy + + @pytest.mark.parametrize( "workflows_enabled, workflow_controller_enabled", [(True, True), (True, False), (False, None), (None, None)], ) -def test_cli_upgrade_2023_5_1_to_2023_7_2( +def test_cli_upgrade_2023_7_1_to_2023_7_2( monkeypatch: pytest.MonkeyPatch, workflows_enabled: bool, workflow_controller_enabled: bool, @@ -123,7 +140,7 @@ def test_cli_upgrade_2023_5_1_to_2023_7_2( upgraded = assert_nebari_upgrade_success( monkeypatch, - "2023.5.1", + "2023.7.1", "2023.7.2", addl_config=addl_config, # Do you want to enable the Nebari Workflow Controller? @@ -149,7 +166,7 @@ def test_cli_upgrade_2023_5_1_to_2023_7_2( def test_cli_upgrade_image_tags(monkeypatch: pytest.MonkeyPatch): start_version = "2023.5.1" - end_version = "2023.7.2" + end_version = "2023.7.1" upgraded = assert_nebari_upgrade_success( monkeypatch, @@ -527,6 +544,7 @@ def assert_nebari_upgrade_success( monkeypatch: pytest.MonkeyPatch, start_version: str, end_version: str, + provider: str = "local", addl_args: List[str] = [], addl_config: Dict[str, Any] = {}, inputs: List[str] = [], @@ -544,7 +562,7 @@ def assert_nebari_upgrade_success( **yaml.safe_load( f""" project_name: test -provider: local +provider: {provider} domain: test.example.com namespace: dev nebari_version: {start_version} From d6dd63bc7219f82a7b2bd44e43d493e1fa6cd928 Mon Sep 17 00:00:00 2001 From: kenafoster Date: Thu, 28 Sep 2023 23:29:39 -0400 Subject: [PATCH 16/17] 2023.9.1 AWS upgrade changes (#2041) --- src/_nebari/upgrade.py | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index e67c7082cc..c621f50b07 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -18,7 +18,7 @@ yaml, ) from _nebari.version import __version__, rounded_ver_parse -from nebari import schema +from nebari.schema import is_version_accepted, ProviderEnum logger = logging.getLogger(__name__) @@ -28,7 +28,7 @@ ARGO_JUPYTER_SCHEDULER_REPO = "https://github.com/nebari-dev/argo-jupyter-scheduler" UPGRADE_KUBERNETES_MESSAGE = "Please see the [green][link=https://www.nebari.dev/docs/how-tos/kubernetes-version-upgrade]Kubernetes upgrade docs[/link][/green] for more information." - +DESTRUCTIVE_UPGRADE_WARNING = "-> This version upgrade will result in your cluster being completely torn down and redeployed. Please ensure you have backed up any data you wish to keep before proceeding!!!" def do_upgrade(config_filename, attempt_fixes=False): config = load_yaml(config_filename) @@ -47,7 +47,7 @@ def do_upgrade(config_filename, attempt_fixes=False): ) return except (ValidationError, ValueError) as e: - if schema.is_version_accepted(config.get("nebari_version", "")): + if is_version_accepted(config.get("nebari_version", "")): # There is an unrelated validation problem rich.print( f"Your config file [purple]{config_filename}[/purple] appears to be already up-to-date for Nebari version [green]{__version__}[/green] but there is another validation error.\n" @@ -494,10 +494,10 @@ def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): provider = config["provider"] - if provider == "aws": + if provider == ProviderEnum.aws.value: rich.print("\n ⚠️ DANGER ⚠️") rich.print( - "-> This version upgrade will result in your cluster being completely torn down and redeployed. Please ensure you have backed up any data you wish to keep before proceeding!!!", + DESTRUCTIVE_UPGRADE_WARNING, "The 'prevent_deploy' flag has been set in your config file and must be manually removed to deploy.", ) config["prevent_deploy"] = True @@ -530,6 +530,9 @@ def _version_specific_upgrade( class Upgrade_2023_9_1(UpgradeStep): version = "2023.9.1" + # JupyterHub Helm chart 2.0.0 (app version 3.0.0) requires K8S Version >=1.23. (reference: https://z2jh.jupyter.org/en/stable/) + # This released has been tested against 1.26 + min_k8s_version = 1.26 def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs @@ -561,7 +564,7 @@ def _version_specific_upgrade( rich.print("-> Removing cdsdashboards from config file.") del config["cdsdashboards"] - # Kubernetes version check. Minimum Kubernetes version is 1.26 + # Kubernetes version check # JupyterHub Helm chart 2.0.0 (app version 3.0.0) requires K8S Version >=1.23. (reference: https://z2jh.jupyter.org/en/stable/) provider = config["provider"] @@ -569,7 +572,7 @@ def _version_specific_upgrade( # Get current Kubernetes version if available in config. current_version = config.get(provider_config_block, {}).get( - "kubernetes_version", "NA" + "kubernetes_version", None ) # Convert to decimal prefix @@ -577,28 +580,28 @@ def _version_specific_upgrade( current_version = get_k8s_version_prefix(current_version) # Try to convert known Kubernetes versions to float. - if not current_version == "NA": + if not current_version == None: try: current_version = float(current_version) - except TypeError: - current_version = "NA" + except ValueError: + current_version = None # Handle checks for when Kubernetes version should be detectable if provider in ["aws", "azure", "gcp", "do"]: # Kubernetes version not found in provider block - if current_version == "NA": + if current_version == None: rich.print("\n ⚠️ Warning ⚠️") rich.print( - f"-> Unable to detect Kubernetes version for provider {provider}. Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26. Please confirm your Kubernetes version is configured before upgrading." + f"-> Unable to detect Kubernetes version for provider {provider}. Nebari version [green]{self.version}[/green] requires Kubernetes version {str(self.min_k8s_version)}. Please confirm your Kubernetes version is configured before upgrading." ) # Kubernetes version less than required minimum - if isinstance(current_version, float) and current_version < 1.26: + if isinstance(current_version, float) and current_version < self.min_k8s_version: rich.print("\n ⚠️ Warning ⚠️") rich.print( - f"-> Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26. Your configured Kubernetes version is [red]{current_version}[/red]. {UPGRADE_KUBERNETES_MESSAGE}" + f"-> Nebari version [green]{self.version}[/green] requires Kubernetes version {str(self.min_k8s_version)}. Your configured Kubernetes version is [red]{current_version}[/red]. {UPGRADE_KUBERNETES_MESSAGE}" ) - version_diff = round(1.26 - current_version, 2) + version_diff = round(self.min_k8s_version - current_version, 2) if version_diff > 0.01: rich.print( "-> The Kubernetes version is multiple minor versions behind the minimum required version. You will need to perform the upgrade one minor version at a time. For example, if your current version is 1.24, you will need to upgrade to 1.25, and then 1.26." @@ -610,7 +613,7 @@ def _version_specific_upgrade( else: rich.print("\n ⚠️ Warning ⚠️") rich.print( - f"-> Unable to detect Kubernetes version for provider {provider}. Nebari version [green]{self.version}[/green] requires Kubernetes version 1.26 or greater." + f"-> Unable to detect Kubernetes version for provider {provider}. Nebari version [green]{self.version}[/green] requires Kubernetes version {str(self.min_k8s_version)} or greater." ) rich.print( "-> Please ensure your Kubernetes version is up-to-date before proceeding." @@ -618,9 +621,7 @@ def _version_specific_upgrade( if provider == "aws": rich.print("\n ⚠️ DANGER ⚠️") - rich.print( - "-> This version upgrade will result in your cluster being completely torn down and redeployed. Please ensure you have backed up any data you wish to keep before proceeding!!!" - ) + rich.print(DESTRUCTIVE_UPGRADE_WARNING) return config From 5d8cde4be5bd826f231d1b59403cd2aac68e58f7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 03:29:49 +0000 Subject: [PATCH 17/17] [pre-commit.ci] Apply automatic pre-commit fixes --- src/_nebari/upgrade.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index c621f50b07..fba60767e1 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -18,7 +18,7 @@ yaml, ) from _nebari.version import __version__, rounded_ver_parse -from nebari.schema import is_version_accepted, ProviderEnum +from nebari.schema import ProviderEnum, is_version_accepted logger = logging.getLogger(__name__) @@ -30,6 +30,7 @@ UPGRADE_KUBERNETES_MESSAGE = "Please see the [green][link=https://www.nebari.dev/docs/how-tos/kubernetes-version-upgrade]Kubernetes upgrade docs[/link][/green] for more information." DESTRUCTIVE_UPGRADE_WARNING = "-> This version upgrade will result in your cluster being completely torn down and redeployed. Please ensure you have backed up any data you wish to keep before proceeding!!!" + def do_upgrade(config_filename, attempt_fixes=False): config = load_yaml(config_filename) if config.get("qhub_version"): @@ -580,7 +581,7 @@ def _version_specific_upgrade( current_version = get_k8s_version_prefix(current_version) # Try to convert known Kubernetes versions to float. - if not current_version == None: + if current_version is not None: try: current_version = float(current_version) except ValueError: @@ -589,14 +590,17 @@ def _version_specific_upgrade( # Handle checks for when Kubernetes version should be detectable if provider in ["aws", "azure", "gcp", "do"]: # Kubernetes version not found in provider block - if current_version == None: + if current_version is None: rich.print("\n ⚠️ Warning ⚠️") rich.print( f"-> Unable to detect Kubernetes version for provider {provider}. Nebari version [green]{self.version}[/green] requires Kubernetes version {str(self.min_k8s_version)}. Please confirm your Kubernetes version is configured before upgrading." ) # Kubernetes version less than required minimum - if isinstance(current_version, float) and current_version < self.min_k8s_version: + if ( + isinstance(current_version, float) + and current_version < self.min_k8s_version + ): rich.print("\n ⚠️ Warning ⚠️") rich.print( f"-> Nebari version [green]{self.version}[/green] requires Kubernetes version {str(self.min_k8s_version)}. Your configured Kubernetes version is [red]{current_version}[/red]. {UPGRADE_KUBERNETES_MESSAGE}" @@ -621,7 +625,7 @@ def _version_specific_upgrade( if provider == "aws": rich.print("\n ⚠️ DANGER ⚠️") - rich.print(DESTRUCTIVE_UPGRADE_WARNING) + rich.print(DESTRUCTIVE_UPGRADE_WARNING) return config