From 701394c3d7463ffb2bd223daf9662921244ad34d Mon Sep 17 00:00:00 2001 From: Isman Firmansyah Date: Mon, 2 Jan 2023 14:07:54 +0700 Subject: [PATCH] fix: modify clnId column type (#3459) * build: update JANS_SOURCE_VERSION and CN_BUILD_DATE * fix(pycloudlib): remove cached_property metadata * fix: modify clnId column type --- docker-jans-auth-server/Dockerfile | 4 +-- docker-jans-config-api/Dockerfile | 4 +-- docker-jans-fido2/Dockerfile | 4 +-- docker-jans-monolith/Dockerfile | 2 +- docker-jans-persistence-loader/Dockerfile | 2 +- .../scripts/spanner_setup.py | 4 +++ .../scripts/sql_setup.py | 32 +++++++++++++++-- docker-jans-scim/Dockerfile | 4 +-- .../jans/pycloudlib/persistence/sql.py | 34 ++++++++++--------- 9 files changed, 61 insertions(+), 29 deletions(-) diff --git a/docker-jans-auth-server/Dockerfile b/docker-jans-auth-server/Dockerfile index 71f891c4dbd..9c7d5b98cc4 100644 --- a/docker-jans-auth-server/Dockerfile +++ b/docker-jans-auth-server/Dockerfile @@ -56,7 +56,7 @@ RUN /opt/jython/bin/pip uninstall -y pip # =========== ENV CN_VERSION=1.0.6-SNAPSHOT -ENV CN_BUILD_DATE='2022-12-25 08:14' +ENV CN_BUILD_DATE='2022-12-29 08:15' ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-auth-server/${CN_VERSION}/jans-auth-server-${CN_VERSION}.war # Install Jans Auth @@ -118,7 +118,7 @@ RUN mkdir -p ${JETTY_BASE}/jans-auth/agama/fl \ ${JETTY_BASE}/jans-auth/agama/scripts # janssenproject/jans SHA commit -ENV JANS_SOURCE_VERSION=bd2cdf8501d60959498078bbb31650965c321c73 +ENV JANS_SOURCE_VERSION=d95fc492ca00ccee0ecc88132376c128203faefb # note that as we're pulling from a monorepo (with multiple project in it) # we are using partial-clone and sparse-checkout to get the agama code diff --git a/docker-jans-config-api/Dockerfile b/docker-jans-config-api/Dockerfile index 7f6e517ae73..f75c86d7e51 100644 --- a/docker-jans-config-api/Dockerfile +++ b/docker-jans-config-api/Dockerfile @@ -43,7 +43,7 @@ RUN wget -q https://maven.jans.io/maven/io/jans/jython-installer/${JYTHON_VERSIO # ========== ENV CN_VERSION=1.0.6-SNAPSHOT -ENV CN_BUILD_DATE='2022-12-25 08:17' +ENV CN_BUILD_DATE='2022-12-29 11:35' ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-config-api-server/${CN_VERSION}/jans-config-api-server-${CN_VERSION}.war # Install Jans Config API @@ -110,7 +110,7 @@ COPY conf/prometheus-config.yaml /opt/prometheus/ # jans-linux-setup sync # ===================== -ENV JANS_SOURCE_VERSION=bd2cdf8501d60959498078bbb31650965c321c73 +ENV JANS_SOURCE_VERSION=d95fc492ca00ccee0ecc88132376c128203faefb ARG JANS_SETUP_DIR=jans-linux-setup/jans_setup ARG JANS_CONFIG_API_RESOURCES=jans-config-api/server/src/main/resources diff --git a/docker-jans-fido2/Dockerfile b/docker-jans-fido2/Dockerfile index 18eeb464fb3..0fb29b68864 100644 --- a/docker-jans-fido2/Dockerfile +++ b/docker-jans-fido2/Dockerfile @@ -35,7 +35,7 @@ EXPOSE 8080 # ===== ENV CN_VERSION=1.0.6-SNAPSHOT -ENV CN_BUILD_DATE='2022-12-25 08:15' +ENV CN_BUILD_DATE='2022-12-29 08:50' ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-fido2-server/${CN_VERSION}/jans-fido2-server-${CN_VERSION}.war # Install FIDO2 @@ -54,7 +54,7 @@ RUN mkdir -p ${JETTY_BASE}/jans-fido2/webapps \ # jans-linux-setup sync # ===================== -ENV JANS_SOURCE_VERSION=bd2cdf8501d60959498078bbb31650965c321c73 +ENV JANS_SOURCE_VERSION=d95fc492ca00ccee0ecc88132376c128203faefb ARG JANS_SETUP_DIR=jans-linux-setup/jans_setup # note that as we're pulling from a monorepo (with multiple project in it) diff --git a/docker-jans-monolith/Dockerfile b/docker-jans-monolith/Dockerfile index 169d6f51246..65a83c7f8fe 100644 --- a/docker-jans-monolith/Dockerfile +++ b/docker-jans-monolith/Dockerfile @@ -38,7 +38,7 @@ EXPOSE 443 8080 1636 # jans-linux-setup # ===================== -ENV JANS_SOURCE_VERSION=bd2cdf8501d60959498078bbb31650965c321c73 +ENV JANS_SOURCE_VERSION=d95fc492ca00ccee0ecc88132376c128203faefb # cleanup RUN rm -rf /tmp/jans diff --git a/docker-jans-persistence-loader/Dockerfile b/docker-jans-persistence-loader/Dockerfile index 149da43880e..772f4b89b25 100644 --- a/docker-jans-persistence-loader/Dockerfile +++ b/docker-jans-persistence-loader/Dockerfile @@ -24,7 +24,7 @@ RUN python3 -m ensurepip \ # ===================== # janssenproject/jans SHA commit -ENV JANS_SOURCE_VERSION=bd2cdf8501d60959498078bbb31650965c321c73 +ENV JANS_SOURCE_VERSION=d95fc492ca00ccee0ecc88132376c128203faefb ARG JANS_SETUP_DIR=jans-linux-setup/jans_setup ARG JANS_SCRIPT_CATALOG_DIR=docs/script-catalog ARG JANS_CONFIG_API_RESOURCES=jans-config-api/server/src/main/resources diff --git a/docker-jans-persistence-loader/scripts/spanner_setup.py b/docker-jans-persistence-loader/scripts/spanner_setup.py index ae676e669ef..ce58cd33f76 100644 --- a/docker-jans-persistence-loader/scripts/spanner_setup.py +++ b/docker-jans-persistence-loader/scripts/spanner_setup.py @@ -421,6 +421,10 @@ def column_int_to_string(table_name, col_name): for mod in [ ("jansPerson", "jansMobileDevices"), ("jansPerson", "jansOTPDevices"), + ("jansToken", "clnId"), + ("jansUmaRPT", "clnId"), + ("jansUmaPCT", "clnId"), + ("jansCibaReq", "clnId"), ]: column_from_array(mod[0], mod[1]) diff --git a/docker-jans-persistence-loader/scripts/sql_setup.py b/docker-jans-persistence-loader/scripts/sql_setup.py index 9c8320f4b81..62f8b17b6c8 100644 --- a/docker-jans-persistence-loader/scripts/sql_setup.py +++ b/docker-jans-persistence-loader/scripts/sql_setup.py @@ -5,6 +5,7 @@ from string import Template from pathlib import Path +from sqlalchemy import text from sqlalchemy.exc import OperationalError from jans.pycloudlib.persistence.sql import SqlClient @@ -322,9 +323,28 @@ def column_from_json(table_name, col_name): for row in self.client.search(table_name, ["doc_id", col_name]) } - # to change the storage format of a JSON column, drop the column and - # add the column back specifying the new storage format with self.client.engine.connect() as conn: + # mysql will raise error if dropping column which has functional index, + # hence the associated index must be dropped first + if self.client.dialect == "mysql": + for idx in conn.execute( + text( + "SELECT index_name " + "FROM information_schema.statistics " + "WHERE table_name = :table_name " + "AND index_name LIKE :index_name '%' " + "AND expression LIKE '%' :col_name '%';" + ), + { + "table_name": table_name, + "index_name": f"{table_name}_json_", + "col_name": col_name + }, + ): + conn.execute(f"ALTER TABLE {table_name} DROP INDEX {idx[0]}") + + # to change the storage format of a JSON column, drop the column and + # add the column back specifying the new storage format conn.execute(f"ALTER TABLE {self.client.quoted_id(table_name)} DROP COLUMN {self.client.quoted_id(col_name)}") conn.execute(f"ALTER TABLE {self.client.quoted_id(table_name)} ADD COLUMN {self.client.quoted_id(col_name)} {data_type}") @@ -333,8 +353,10 @@ def column_from_json(table_name, col_name): # pre-populate the modified column for doc_id, value in values.items(): - if value and "v" in value and value["v"]: + if self.client.dialect == "mysql" and value and value.get("v", []): new_value = value["v"][0] + elif self.client.dialect == "pgsql" and value: + new_value = value[0] else: new_value = "" self.client.update(table_name, doc_id, {col_name: new_value}) @@ -427,6 +449,10 @@ def column_from_json(table_name, col_name): for mod in [ ("jansPerson", "jansMobileDevices"), ("jansPerson", "jansOTPDevices"), + ("jansToken", "clnId"), + ("jansUmaRPT", "clnId"), + ("jansUmaPCT", "clnId"), + ("jansCibaReq", "clnId"), ]: column_from_json(mod[0], mod[1]) diff --git a/docker-jans-scim/Dockerfile b/docker-jans-scim/Dockerfile index 047c9288e66..d2ec607fde2 100644 --- a/docker-jans-scim/Dockerfile +++ b/docker-jans-scim/Dockerfile @@ -46,7 +46,7 @@ RUN wget -q https://maven.jans.io/maven/io/jans/jython-installer/${JYTHON_VERSIO # ==== ENV CN_VERSION=1.0.6-SNAPSHOT -ENV CN_BUILD_DATE='2022-12-25 08:16' +ENV CN_BUILD_DATE='2022-12-29 11:33' ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-scim-server/${CN_VERSION}/jans-scim-server-${CN_VERSION}.war # Install SCIM @@ -80,7 +80,7 @@ COPY conf/prometheus-config.yaml /opt/prometheus/ # jans-linux-setup sync # ===================== -ENV JANS_SOURCE_VERSION=bd2cdf8501d60959498078bbb31650965c321c73 +ENV JANS_SOURCE_VERSION=d95fc492ca00ccee0ecc88132376c128203faefb ARG JANS_SETUP_DIR=jans-linux-setup/jans_setup ARG JANS_SCIM_RESOURCE_DIR=jans-scim/server/src/main/resources diff --git a/jans-pycloudlib/jans/pycloudlib/persistence/sql.py b/jans-pycloudlib/jans/pycloudlib/persistence/sql.py index b3b4fa5e940..168d2b469e2 100644 --- a/jans-pycloudlib/jans/pycloudlib/persistence/sql.py +++ b/jans-pycloudlib/jans/pycloudlib/persistence/sql.py @@ -258,6 +258,7 @@ def __init__(self, manager: Manager, *args: _t.Any, **kwargs: _t.Any) -> None: self.adapter = MysqlAdapter() self.dialect = self.adapter.dialect + self._metadata: _t.Optional[MetaData] = None @cached_property def engine(self) -> Engine: @@ -274,24 +275,25 @@ def engine_url(self) -> str: password = get_sql_password(self.manager) return f"{self.adapter.connector}://{user}:{password}@{host}:{port}/{database}" - @cached_property + @property def metadata(self) -> MetaData: """Lazy init of metadata.""" - with warnings.catch_warnings(): - # postgresql driver will show warnings about unsupported reflection - # on expression-based index, i.e. `lower(uid::text)`; but we don't - # want to clutter the logs with these warnings, hence we suppress the - # warnings - warnings.filterwarnings( - "ignore", - message="Skipped unsupported reflection of expression-based index", - category=SAWarning, - ) - - # do reflection on database table - metadata = MetaData(bind=self.engine) - metadata.reflect() - return metadata + if not self._metadata: + with warnings.catch_warnings(): + # postgresql driver will show warnings about unsupported reflection + # on expression-based index, i.e. `lower(uid::text)`; but we don't + # want to clutter the logs with these warnings, hence we suppress the + # warnings + warnings.filterwarnings( + "ignore", + message="Skipped unsupported reflection of expression-based index", + category=SAWarning, + ) + + # do reflection on database table + self._metadata = MetaData(bind=self.engine) + self._metadata.reflect() + return self._metadata def connected(self) -> bool: """Check whether connection is alive by executing simple query."""