From b22f51d25ff773e8a94bf057907b503fb8abeb10 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Thu, 28 Aug 2025 16:39:13 -0300 Subject: [PATCH 01/11] #246: Use SLC 10.0.0 --- doc/changes/unreleased.md | 3 ++- exasol/nb_connector/slc/constants.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 9ce161ea..7f123f15 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -4,4 +4,5 @@ - #244: Updated PTB to 1.9.0 - #247: Start ITDE with disabled certificate check for importer - - #242: Refactoring of install_model \ No newline at end of file + - #242: Refactoring of install_model + - #246: Use SLC 10.0.0 \ No newline at end of file diff --git a/exasol/nb_connector/slc/constants.py b/exasol/nb_connector/slc/constants.py index 6ecafae6..6629e76b 100644 --- a/exasol/nb_connector/slc/constants.py +++ b/exasol/nb_connector/slc/constants.py @@ -16,9 +16,9 @@ FLAVORS_PATH_IN_SLC_REPO = Path("flavors") """Path to flavors within the script-languages-release repository""" -SLC_RELEASE_TAG = "9.7.0" +SLC_RELEASE_TAG = "10.0.0" """ -Using the SLC_RELEASE 9.7.0 because we are limited to slc-tool 3.*. (see pyproject.toml) +Using the SLC_RELEASE 10.0.0 because we are limited to slc-tool 3.*. (see pyproject.toml) Check the developer guide (./doc/developer-guide.md) for more information. """ From 07a089d45a1e30cb2fdf6d9c92c9b6ec53aa5b1a Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 1 Sep 2025 10:21:08 -0300 Subject: [PATCH 02/11] Prepare release 2.0.0 --- doc/changes/changelog.md | 2 ++ doc/changes/changes_2.0.0.md | 12 ++++++++++++ doc/changes/unreleased.md | 7 ------- pyproject.toml | 2 +- version.py | 2 +- 5 files changed, 16 insertions(+), 9 deletions(-) create mode 100644 doc/changes/changes_2.0.0.md diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index 8bc25296..e063e9f1 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,6 +1,7 @@ # Changes * [unreleased](unreleased) +* [2.0.0](changes_2.0.0.md) * [1.0.0](changes_1.0.0.md) * [0.6.0](changes_0.6.0.md) * [0.5.1](changes_0.5.1) @@ -28,6 +29,7 @@ hidden: --- unreleased +changes_2.0.0 changes_1.0.0 changes_0.6.0 changes_0.5.0 diff --git a/doc/changes/changes_2.0.0.md b/doc/changes/changes_2.0.0.md new file mode 100644 index 00000000..f09eeb33 --- /dev/null +++ b/doc/changes/changes_2.0.0.md @@ -0,0 +1,12 @@ +# 2.0.0 - 2025-09-01 +## Refactorings + + - #244: Updated PTB to 1.9.0 + - #247: Start ITDE with disabled certificate check for importer + - #242: Refactoring of install_model + - #246: Use SLC 10.0.0 + +## Dependency Updates + +### `dev` +* Updated dependency `exasol-toolbox:1.7.4` to `1.9.0` diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 7f123f15..79e701b8 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -1,8 +1 @@ # Unreleased - -## Refactorings - - - #244: Updated PTB to 1.9.0 - - #247: Start ITDE with disabled certificate check for importer - - #242: Refactoring of install_model - - #246: Use SLC 10.0.0 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index e8ac4b72..08e2acd5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "exasol-notebook-connector" -version = "1.0.0" +version = "2.0.0" requires-python = ">=3.10,<3.13" description = "Components, tools, APIs, and configurations in order to connect Jupyter notebooks to Exasol and various other systems." packages = [{ include = "exasol" }, ] diff --git a/version.py b/version.py index e3dc1024..bf9696ec 100644 --- a/version.py +++ b/version.py @@ -8,7 +8,7 @@ `poetry version X.Y.Z`. """ -MAJOR = 1 +MAJOR = 2 MINOR = 0 PATCH = 0 VERSION = f"{MAJOR}.{MINOR}.{PATCH}" From ee9d4819684e368d0433eb94abd3ee1664ddeaa3 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 1 Sep 2025 11:24:18 -0300 Subject: [PATCH 03/11] #250: Added export function which does not copy container file to export directory --- doc/changes/changes_2.0.0.md | 8 ++++++++ .../nb_connector/slc/script_language_container.py | 14 +++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/doc/changes/changes_2.0.0.md b/doc/changes/changes_2.0.0.md index f09eeb33..e5f17c81 100644 --- a/doc/changes/changes_2.0.0.md +++ b/doc/changes/changes_2.0.0.md @@ -1,4 +1,8 @@ # 2.0.0 - 2025-09-01 + +This release disables the certificate check for the importer when starting a docker db using the ITDE (this is a temporary fix until the notebook connector updated to pyexasol >=1.0.0). Also, the installation of AI models has been refactored which resulted in breaking API changes. The used Script-Language-Container version was updated to 10.0.0. Besides, there were internal improvements. + + ## Refactorings - #244: Updated PTB to 1.9.0 @@ -6,6 +10,10 @@ - #242: Refactoring of install_model - #246: Use SLC 10.0.0 +## Features + + - #250: Added export function which does not copy container file to export directory + ## Dependency Updates ### `dev` diff --git a/exasol/nb_connector/slc/script_language_container.py b/exasol/nb_connector/slc/script_language_container.py index 1cf742e3..2083ae12 100644 --- a/exasol/nb_connector/slc/script_language_container.py +++ b/exasol/nb_connector/slc/script_language_container.py @@ -163,7 +163,7 @@ def custom_conda_file(self) -> Path: """ return self.custom_packages_dir / "conda_packages" - def export(self): + def export(self) -> None: """ Exports the current SLC to the export directory. """ @@ -176,6 +176,18 @@ def export(self): compression_strategy=self.compression_strategy, ) + def export_no_copy(self) -> None: + """ + Exports the current SLC to the internal output directory only, without copying to the export directory. + """ + with current_directory(self.checkout_dir): + exaslct_api.export( + flavor_path=(str(self._flavor_path_rel),), + output_directory=str(self.workspace.output_path), + release_name=self.language_alias, + compression_strategy=self.compression_strategy, + ) + def deploy(self): """ Deploys the current script-languages-container to the database and From f62adcafc434dd537402860c9c5fc343bb164b2d Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 1 Sep 2025 11:46:40 -0300 Subject: [PATCH 04/11] Fixed itest_slct_manager_with_gpu.py --- test/integration/gpu/itest_slct_manager_with_gpu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/gpu/itest_slct_manager_with_gpu.py b/test/integration/gpu/itest_slct_manager_with_gpu.py index e46bc47d..e1c1f47d 100644 --- a/test/integration/gpu/itest_slct_manager_with_gpu.py +++ b/test/integration/gpu/itest_slct_manager_with_gpu.py @@ -46,7 +46,7 @@ def sample_slc(secrets_module: Secrets) -> ScriptLanguageContainer: @pytest.fixture def custom_packages() -> list[tuple[str, str]]: - return [("numba[cuda]", "0.61.2")] + return [("numba", "0.61.2")] @pytest.mark.dependency(name="append_custom_packages") From 1dbe7c969a3614c839351dbd33a6bb896e09b744 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 1 Sep 2025 13:00:25 -0300 Subject: [PATCH 05/11] Added integration test --- test/integration/ordinary/itest_slc.py | 31 ++++++++++++++++++++------ 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/test/integration/ordinary/itest_slc.py b/test/integration/ordinary/itest_slc.py index 6797f3e9..90d9ea0a 100644 --- a/test/integration/ordinary/itest_slc.py +++ b/test/integration/ordinary/itest_slc.py @@ -94,8 +94,30 @@ def other_slc( def custom_packages() -> list[tuple[str, str, str]]: return [("xgboost", "2.0.3", "xgboost"), ("scikit-learn", "1.5.0", "sklearn")] +def _check_exported_slc_exists(expected_suffix: str, expected_path: Path) -> None: + tar = [f for f in expected_path.glob(f"*.{expected_suffix}")] + assert len(tar) == 1 + assert tar[0].is_file() + tar_sum = [f for f in expected_path.glob(f"*.{expected_suffix}.sha512sum")] + assert len(tar_sum) == 1 + assert tar_sum[0].is_file() + -@pytest.mark.dependency(name="export_slc") +@pytest.mark.dependency(name="export_slc_no_copy") +def test_export_slc_no_copy( + sample_slc: ScriptLanguageContainer, compression_strategy: CompressionStrategy +): + sample_slc.export_no_copy() + export_path = sample_slc.workspace.export_path + expected_suffix = ( + "tar" if compression_strategy == CompressionStrategy.NONE else "tar.gz" + ) + assert not export_path.exists() + + internal_export_path = sample_slc.workspace.output_path / "cache" / "exports" + _check_exported_slc_exists(expected_suffix, internal_export_path) + +@pytest.mark.dependency(name="export_slc", depends=["export_slc_no_copy"]) def test_export_slc( sample_slc: ScriptLanguageContainer, compression_strategy: CompressionStrategy ): @@ -105,12 +127,7 @@ def test_export_slc( "tar" if compression_strategy == CompressionStrategy.NONE else "tar.gz" ) assert export_path.exists() - tar = [f for f in export_path.glob(f"*.{expected_suffix}")] - assert len(tar) == 1 - assert tar[0].is_file() - tar_sum = [f for f in export_path.glob(f"*.{expected_suffix}.sha512sum")] - assert len(tar_sum) == 1 - assert tar_sum[0].is_file() + _check_exported_slc_exists(expected_suffix, export_path) def slc_docker_tag_prefix(slc: ScriptLanguageContainer) -> str: From 5f1aae2b2f08ace7e9b49771731fa0ef0365821f Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 1 Sep 2025 13:00:43 -0300 Subject: [PATCH 06/11] Updated release date --- doc/changes/changes_2.0.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/changes/changes_2.0.0.md b/doc/changes/changes_2.0.0.md index e5f17c81..f413d134 100644 --- a/doc/changes/changes_2.0.0.md +++ b/doc/changes/changes_2.0.0.md @@ -1,4 +1,4 @@ -# 2.0.0 - 2025-09-01 +# 2.0.0 - 2025-09-02 This release disables the certificate check for the importer when starting a docker db using the ITDE (this is a temporary fix until the notebook connector updated to pyexasol >=1.0.0). Also, the installation of AI models has been refactored which resulted in breaking API changes. The used Script-Language-Container version was updated to 10.0.0. Besides, there were internal improvements. From c20e0213bd9ef5f6f6a2d98c2104ea58a675aebd Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 1 Sep 2025 13:13:15 -0300 Subject: [PATCH 07/11] Fixed formatting --- test/integration/ordinary/itest_slc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/integration/ordinary/itest_slc.py b/test/integration/ordinary/itest_slc.py index 90d9ea0a..b4c8eab2 100644 --- a/test/integration/ordinary/itest_slc.py +++ b/test/integration/ordinary/itest_slc.py @@ -94,6 +94,7 @@ def other_slc( def custom_packages() -> list[tuple[str, str, str]]: return [("xgboost", "2.0.3", "xgboost"), ("scikit-learn", "1.5.0", "sklearn")] + def _check_exported_slc_exists(expected_suffix: str, expected_path: Path) -> None: tar = [f for f in expected_path.glob(f"*.{expected_suffix}")] assert len(tar) == 1 @@ -117,6 +118,7 @@ def test_export_slc_no_copy( internal_export_path = sample_slc.workspace.output_path / "cache" / "exports" _check_exported_slc_exists(expected_suffix, internal_export_path) + @pytest.mark.dependency(name="export_slc", depends=["export_slc_no_copy"]) def test_export_slc( sample_slc: ScriptLanguageContainer, compression_strategy: CompressionStrategy From cf2143c2c5621a0a3dfd6d07703eeb41023848ba Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 1 Sep 2025 14:04:18 -0300 Subject: [PATCH 08/11] Deactivated SaaS Tests --- .github/workflows/merge-gate.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/merge-gate.yml b/.github/workflows/merge-gate.yml index ee11b56e..7fdd13f0 100644 --- a/.github/workflows/merge-gate.yml +++ b/.github/workflows/merge-gate.yml @@ -33,13 +33,13 @@ jobs: permissions: contents: read - saas-tests: - name: SaaS - needs: [ run-slow-tests ] - uses: ./.github/workflows/tests-saas.yml - secrets: inherit - permissions: - contents: read +# saas-tests: +# name: SaaS +# needs: [ run-slow-tests ] +# uses: ./.github/workflows/tests-saas.yml +# secrets: inherit +# permissions: +# contents: read large-runner-tests: name: Large Runner From b36e5fedd2c34d850b2b91699fe1787883529fb9 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 2 Sep 2025 06:12:00 -0300 Subject: [PATCH 09/11] Update doc/changes/changes_2.0.0.md Co-authored-by: Ariel Schulz <43442541+ArBridgeman@users.noreply.github.com> --- doc/changes/changes_2.0.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/changes/changes_2.0.0.md b/doc/changes/changes_2.0.0.md index f413d134..45865e97 100644 --- a/doc/changes/changes_2.0.0.md +++ b/doc/changes/changes_2.0.0.md @@ -1,6 +1,6 @@ # 2.0.0 - 2025-09-02 -This release disables the certificate check for the importer when starting a docker db using the ITDE (this is a temporary fix until the notebook connector updated to pyexasol >=1.0.0). Also, the installation of AI models has been refactored which resulted in breaking API changes. The used Script-Language-Container version was updated to 10.0.0. Besides, there were internal improvements. +This release disables the certificate check for the importer when starting a docker db using the ITDE (this is a temporary fix until the notebook connector has been updated to pyexasol >=1.0.0). Also, the installation of AI models has been refactored which resulted in breaking API changes. The used Script-Language-Container version was updated to 10.0.0. Otherwise, there were internal improvements. ## Refactorings From e87b6adba633ef498e965c6ba33e091e13804c35 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 2 Sep 2025 06:12:31 -0300 Subject: [PATCH 10/11] Added comment to commented Saas integration tests --- .github/workflows/merge-gate.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/merge-gate.yml b/.github/workflows/merge-gate.yml index 7fdd13f0..6234dbe5 100644 --- a/.github/workflows/merge-gate.yml +++ b/.github/workflows/merge-gate.yml @@ -33,6 +33,8 @@ jobs: permissions: contents: read +# Deactivated because SaaS Tests are broken because of breaking API changes in SaaS Rest API +# See https://github.com/exasol/notebook-connector/issues/251 # saas-tests: # name: SaaS # needs: [ run-slow-tests ] From 071aaaaf6e333a642b99db4d7ca7bfc064dad6ad Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 2 Sep 2025 10:48:51 -0300 Subject: [PATCH 11/11] #252: Fixed db parameter etlCheckCertsDefault --- doc/changes/changes_2.0.0.md | 4 ++++ exasol/nb_connector/itde_manager.py | 2 +- test/unit/test_itde_manager.py | 6 +++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/doc/changes/changes_2.0.0.md b/doc/changes/changes_2.0.0.md index 45865e97..906ec655 100644 --- a/doc/changes/changes_2.0.0.md +++ b/doc/changes/changes_2.0.0.md @@ -14,6 +14,10 @@ This release disables the certificate check for the importer when starting a doc - #250: Added export function which does not copy container file to export directory +## Bugs + + - #252: Fixed db parameter etlCheckCertsDefault + ## Dependency Updates ### `dev` diff --git a/exasol/nb_connector/itde_manager.py b/exasol/nb_connector/itde_manager.py index a549f81d..261a2f47 100644 --- a/exasol/nb_connector/itde_manager.py +++ b/exasol/nb_connector/itde_manager.py @@ -84,7 +84,7 @@ def bring_itde_up(conf: Secrets, env_info: Optional[EnvironmentInfo] = None) -> docker_runtime = None docker_environment_variable: tuple[str, ...] = () - additional_db_parameter: tuple[str, ...] = ("-etlCheckCertsDefault=1",) + additional_db_parameter: tuple[str, ...] = ("-etlCheckCertsDefault=0",) if accelerator == Accelerator.nvidia.value: docker_runtime = "nvidia" docker_environment_variable = ("NVIDIA_VISIBLE_DEVICES=all",) diff --git a/test/unit/test_itde_manager.py b/test/unit/test_itde_manager.py index b85607ee..d835c639 100644 --- a/test/unit/test_itde_manager.py +++ b/test/unit/test_itde_manager.py @@ -72,7 +72,7 @@ def test_bring_itde_up(mock_spawn_env, secrets, env_info, db_image_version): docker_db_image_version=db_image_version, docker_runtime=None, docker_environment_variable=(), - additional_db_parameter=("-etlCheckCertsDefault=1",), + additional_db_parameter=("-etlCheckCertsDefault=0",), log_level=logging.getLevelName(logging.INFO), ) ] @@ -98,13 +98,13 @@ def test_bring_itde_up(mock_spawn_env, secrets, env_info, db_image_version): @pytest.mark.parametrize( "accelerator, expected_docker_runtime, expected_docker_environment_variable, expected_additional_db_parameter", [ - (Accelerator.none.value, None, (), ("-etlCheckCertsDefault=1",)), + (Accelerator.none.value, None, (), ("-etlCheckCertsDefault=0",)), ( Accelerator.nvidia.value, "nvidia", ("NVIDIA_VISIBLE_DEVICES=all",), ( - "-etlCheckCertsDefault=1", + "-etlCheckCertsDefault=0", "-enableAcceleratorDeviceDetection=1", ), ),