Passing the secret store as a parameter

exasol · Feb 2, 2024 · 78914b2 · 78914b2
1 parent 4529e68
commit 78914b2
Show file tree

Hide file tree

Showing 15 changed files with 86 additions and 103 deletions.
diff --git a/doc/changes/changes_0.2.0.md b/doc/changes/changes_0.2.0.md
@@ -18,5 +18,6 @@ Version: 0.2.0
 
 ## Refactoring
 * #160: Implemented the PM's recommendations of 2024-01-24.
+* #120: Passing the secret store object (sb_config) as a parameter to all functions that need it.
 
 ## Documentation
diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/main_config.ipynb b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/main_config.ipynb
@@ -75,7 +75,7 @@
  "metadata": {},
  "outputs": [],
  "source": [
- "display(get_db_selection_ui())"
+ "display(get_db_selection_ui(sb_config))"
  ]
  },
  {
@@ -95,7 +95,7 @@
  "metadata": {},
  "outputs": [],
  "source": [
- "display(get_db_config_ui())"
+ "display(get_db_config_ui(sb_config))"
  ]
  },
  {
@@ -117,7 +117,7 @@
  "metadata": {},
  "outputs": [],
  "source": [
- "display(get_start_docker_db_ui())"
+ "display(get_start_docker_db_ui(sb_config))"
  ]
  },
  {

diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/sagemaker/sme_init.ipynb b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/sagemaker/sme_init.ipynb
@@ -58,7 +58,7 @@
  "outputs": [],
  "source": [
  "%run utils/sme_init_ui.ipynb\n",
- "display(get_sme_config_ui())"
+ "display(get_sme_config_ui(sb_config))"
  ]
  },
  {

diff --git a/...ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/sagemaker/utils/sme_init_ui.ipynb b/...ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/sagemaker/utils/sme_init_ui.ipynb
@@ -35,20 +35,21 @@
  "from enum import Enum\n",
  "import ipywidgets as widgets\n",
  "\n",
+ "from exasol.secret_store import Secrets\n",
  "from exasol.ai_lab_config import AILabConfig as CKey\n",
  "\n",
  "\n",
- "def get_sme_config_ui() -> widgets.Widget:\n",
+ "def get_sme_config_ui(conf: Secrets) -> widgets.Widget:\n",
  "\n",
  " inputs = [\n",
- " ('Access Key ID', widgets.Password(value=sb_config.get(CKey.aws_access_key_id)), CKey.aws_access_key_id),\n",
- " ('Secret Access Key', widgets.Password(value=sb_config.get(CKey.aws_secret_access_key)), CKey.aws_secret_access_key),\n",
- " ('Region', widgets.Text(value=sb_config.get(CKey.aws_region)), CKey.aws_region),\n",
- " ('Sagemaker AWS Role', widgets.Text(value=sb_config.get(CKey.sme_aws_role)), CKey.sme_aws_role),\n",
- " ('Sagemaker S3 Bucket', widgets.Text(value=sb_config.get(CKey.sme_aws_bucket)), CKey.sme_aws_bucket),\n",
+ " ('Access Key ID', widgets.Password(value=conf.get(CKey.aws_access_key_id)), CKey.aws_access_key_id),\n",
+ " ('Secret Access Key', widgets.Password(value=conf.get(CKey.aws_secret_access_key)), CKey.aws_secret_access_key),\n",
+ " ('Region', widgets.Text(value=conf.get(CKey.aws_region)), CKey.aws_region),\n",
+ " ('Sagemaker AWS Role', widgets.Text(value=conf.get(CKey.sme_aws_role)), CKey.sme_aws_role),\n",
+ " ('Sagemaker S3 Bucket', widgets.Text(value=conf.get(CKey.sme_aws_bucket)), CKey.sme_aws_bucket),\n",
  " ]\n",
  "\n",
- " return get_generic_config_ui(sb_config, [inputs], ['Set AWS Credentials'])\n"
+ " return get_generic_config_ui(conf, [inputs], ['Set AWS Credentials'])\n"
  ]
  }
  ],

diff --git a/.../sandbox/runtime/ansible/roles/jupyter/files/notebook/transformers/masked_modelling.ipynb b/.../sandbox/runtime/ansible/roles/jupyter/files/notebook/transformers/masked_modelling.ipynb
@@ -89,7 +89,7 @@
  "outputs": [],
  "source": [
  "%run utils/model_retrieval.ipynb\n",
- "load_huggingface_model(MODEL_NAME)"
+ "load_huggingface_model(sb_config, MODEL_NAME)"
  ]
  },
  {

diff --git a/...andbox/runtime/ansible/roles/jupyter/files/notebook/transformers/question_answering.ipynb b/...andbox/runtime/ansible/roles/jupyter/files/notebook/transformers/question_answering.ipynb
@@ -89,7 +89,7 @@
  "outputs": [],
  "source": [
  "%run utils/model_retrieval.ipynb\n",
- "load_huggingface_model(MODEL_NAME)"
+ "load_huggingface_model(sb_config, MODEL_NAME)"
  ]
  },
  {

diff --git a/...x/runtime/ansible/roles/jupyter/files/notebook/transformers/sequence_classification.ipynb b/...x/runtime/ansible/roles/jupyter/files/notebook/transformers/sequence_classification.ipynb
@@ -90,7 +90,7 @@
  "outputs": [],
  "source": [
  "%run utils/model_retrieval.ipynb\n",
- "load_huggingface_model(MODEL_NAME, method='udf')"
+ "load_huggingface_model(sb_config, MODEL_NAME, method='udf')"
  ]
  },
  {

diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/transformers/te_init.ipynb b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/transformers/te_init.ipynb
@@ -46,7 +46,7 @@
  "outputs": [],
  "source": [
  "%run utils/te_init_ui.ipynb\n",
- "display(get_te_config_ui())"
+ "display(get_te_config_ui(sb_config))"
  ]
  },
  {

diff --git a/...s/sandbox/runtime/ansible/roles/jupyter/files/notebook/transformers/text_generation.ipynb b/...s/sandbox/runtime/ansible/roles/jupyter/files/notebook/transformers/text_generation.ipynb
@@ -71,7 +71,7 @@
  "outputs": [],
  "source": [
  "%run utils/model_retrieval.ipynb\n",
- "load_huggingface_model(MODEL_NAME)"
+ "load_huggingface_model(sb_config, MODEL_NAME)"
  ]
  },
  {

diff --git a/...dbox/runtime/ansible/roles/jupyter/files/notebook/transformers/token_classification.ipynb b/...dbox/runtime/ansible/roles/jupyter/files/notebook/transformers/token_classification.ipynb
@@ -89,7 +89,7 @@
  "outputs": [],
  "source": [
  "%run utils/model_retrieval.ipynb\n",
- "load_huggingface_model(MODEL_NAME)"
+ "load_huggingface_model(sb_config, MODEL_NAME)"
  ]
  },
  {

diff --git a/...ol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/transformers/translation.ipynb b/...ol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/transformers/translation.ipynb
@@ -80,7 +80,6 @@
  "execution_count": null,
  "id": "50a8cc23-bba5-4497-8947-9adc8c83a76c",
  "metadata": {
- "editable": false,
  "slideshow": {
  "slide_type": ""
  },
@@ -91,7 +90,7 @@
  "outputs": [],
  "source": [
  "%run utils/model_retrieval.ipynb\n",
- "load_huggingface_model(MODEL_NAME)"
+ "load_huggingface_model(sb_config, MODEL_NAME)"
  ]
  },
  {

diff --git a/...box/runtime/ansible/roles/jupyter/files/notebook/transformers/utils/model_retrieval.ipynb b/...box/runtime/ansible/roles/jupyter/files/notebook/transformers/utils/model_retrieval.ipynb
@@ -36,12 +36,14 @@
  "metadata": {},
  "outputs": [],
  "source": [
- "def load_huggingface_model_udf(model_name: str) -> None:\n",
+ "from exasol.secret_store import Secrets\n",
+ "\n",
+ "\n",
+ "def load_huggingface_model_udf(conf: Secrets, model_name: str) -> None:\n",
  " \"\"\"\n",
  " Loads specified model into BucketFS using a UDF provided with the Transformer Extension.\n",
- " There must be a global variable - sb_config - providing access to the secret store holding\n",
- " the configuration data.\n",
  "\n",
+ " conf - Access to the secret store holding the configuration data.\n",
  " model_name - Name of the model at Huggingface hub, e.g. facebook/nllb-moe-54b.\n",
  " \"\"\"\n",
  "\n",
@@ -50,16 +52,16 @@
  " from exasol.ai_lab_config import AILabConfig as CKey\n",
  "\n",
  " sql = f\"\"\"\n",
- " SELECT {sb_config.get(CKey.db_schema)}.TE_MODEL_DOWNLOADER_UDF(\n",
+ " SELECT {conf.get(CKey.db_schema)}.TE_MODEL_DOWNLOADER_UDF(\n",
  " '{model_name}',\n",
- " '{sb_config.get(CKey.te_models_bfs_dir)}',\n",
- " '{sb_config.get(CKey.te_bfs_connection)}',\n",
- " '{sb_config.get(CKey.te_hf_connection)}'\n",
+ " '{conf.get(CKey.te_models_bfs_dir)}',\n",
+ " '{conf.get(CKey.te_bfs_connection)}',\n",
+ " '{conf.get(CKey.te_hf_connection)}'\n",
  " )\n",
  " \"\"\"\n",
  "\n",
- " with open_pyexasol_connection(sb_config, compression=True) as conn:\n",
- " conn.execute(query=get_activation_sql(sb_config))\n",
+ " with open_pyexasol_connection(conf, compression=True) as conn:\n",
+ " conn.execute(query=get_activation_sql(conf))\n",
  " conn.execute(query=sql)"
  ]
  },
@@ -80,12 +82,11 @@
  "metadata": {},
  "outputs": [],
  "source": [
- "def load_huggingface_model_cli(model_name: str, force_download: bool = False) -> None:\n",
+ "def load_huggingface_model_cli(conf: Secrets, model_name: str, force_download: bool = False) -> None:\n",
  " \"\"\"\n",
  " Loads specified model into BucketFS by saving it first to a local drive, as per the command-line interface.\n",
- " There must be a global variable - sb_config - providing access to the secret store holding\n",
- " the configuration data.\n",
  "\n",
+ " conf - Access to the secret store holding the configuration data.\n",
  " model_name - Name of the model at Huggingface hub, e.g. facebook/nllb-moe-54b.\n",
  " force_download - If True the model will be reloaded from the hub even if it has been cached before.\n",
  " \"\"\"\n",
@@ -97,11 +98,11 @@
  "\n",
  " # Make a name for the model sub-directory\n",
  " sub_dir = re.sub(r\"[/\\\\?%*:|\\\"<>\\x7F\\x00-\\x1F]\", \"-\", model_name)\n",
- " models_dir = sb_config.get(CKey.te_models_cache_dir)\n",
+ " models_dir = conf.get(CKey.te_models_cache_dir)\n",
  " cache_dir = str(Path(models_dir) / sub_dir)\n",
  "\n",
  " # Upload it to the BucketFS\n",
- " upload_model(sb_config, model_name=model_name, cache_dir=cache_dir, force_download=force_download)"
+ " upload_model(conf, model_name=model_name, cache_dir=cache_dir, force_download=force_download)"
  ]
  },
  {
@@ -121,10 +122,11 @@
  "metadata": {},
  "outputs": [],
  "source": [
- "def load_huggingface_model(model_name: str, method: str = 'cli', force_download: bool = False) -> None:\n",
+ "def load_huggingface_model(conf: Secrets, model_name: str, method: str = 'cli', force_download: bool = False) -> None:\n",
  " \"\"\"\n",
  " Loads specified model into BucketFS choosing one of the two available methods.\n",
  "\n",
+ " conf - Access to the secret store holding the configuration data.\n",
  " model_name - Name of the model at Huggingface hub, e.g. facebook/nllb-moe-54b.\n",
  " method - The recognized values are \"udf\" and \"cli\". If the parameter is not set then\n",
  " will look for method selection in the configuration. That failed the CLI method\n",
@@ -134,9 +136,9 @@
  "\n",
  " method = method.lower()\n",
  " if method == 'udf':\n",
- " load_huggingface_model_udf(model_name)\n",
+ " load_huggingface_model_udf(conf, model_name)\n",
  " else:\n",
- " load_huggingface_model_cli(model_name, force_download=force_download)"
+ " load_huggingface_model_cli(conf, model_name, force_download=force_download)"
  ]
  },
  {

diff --git a/.../sandbox/runtime/ansible/roles/jupyter/files/notebook/transformers/utils/te_init_ui.ipynb b/.../sandbox/runtime/ansible/roles/jupyter/files/notebook/transformers/utils/te_init_ui.ipynb
@@ -34,16 +34,17 @@
  "source": [
  "import ipywidgets as widgets\n",
  "\n",
+ "from exasol.secret_store import Secrets\n",
  "from exasol.ai_lab_config import AILabConfig as CKey\n",
  "\n",
  "\n",
- "def get_te_config_ui() -> widgets.Widget:\n",
+ "def get_te_config_ui(conf: Secrets) -> widgets.Widget:\n",
  "\n",
  " inputs = [\n",
  " ('Access token', widgets.Password(value=sb_config.get(CKey.huggingface_token)), CKey.huggingface_token)\n",
  " ]\n",
  "\n",
- " return get_generic_config_ui(sb_config, [inputs], ['Huggingface Access Parameters'])\n"
+ " return get_generic_config_ui(conf, [inputs], ['Huggingface Access Parameters'])\n"
  ]
  }
  ],

diff --git a/.../runtime/ansible/roles/jupyter/files/notebook/transformers/zero_shot_classification.ipynb b/.../runtime/ansible/roles/jupyter/files/notebook/transformers/zero_shot_classification.ipynb
@@ -89,7 +89,7 @@
  "outputs": [],
  "source": [
  "%run utils/model_retrieval.ipynb\n",
- "load_huggingface_model(MODEL_NAME)"
+ "load_huggingface_model(sb_config, MODEL_NAME)"
  ]
  },
  {