fix: modify clnId column type (#3459)

* build: update JANS_SOURCE_VERSION and CN_BUILD_DATE * fix(pycloudlib): remove cached_property metadata * fix: modify clnId column type
JanssenProject · Jan 2, 2023 · 701394c · 701394c
1 parent 8d8a0a8
commit 701394c
Show file tree

Hide file tree

Showing 9 changed files with 61 additions and 29 deletions.
diff --git a/docker-jans-auth-server/Dockerfile b/docker-jans-auth-server/Dockerfile
@@ -56,7 +56,7 @@ RUN /opt/jython/bin/pip uninstall -y pip
 # ===========
 
 ENV CN_VERSION=1.0.6-SNAPSHOT
-ENV CN_BUILD_DATE='2022-12-25 08:14'
+ENV CN_BUILD_DATE='2022-12-29 08:15'
 ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-auth-server/${CN_VERSION}/jans-auth-server-${CN_VERSION}.war
 
 # Install Jans Auth
@@ -118,7 +118,7 @@ RUN mkdir -p ${JETTY_BASE}/jans-auth/agama/fl \
     ${JETTY_BASE}/jans-auth/agama/scripts
 
 # janssenproject/jans SHA commit
-ENV JANS_SOURCE_VERSION=bd2cdf8501d60959498078bbb31650965c321c73
+ENV JANS_SOURCE_VERSION=d95fc492ca00ccee0ecc88132376c128203faefb
 
 # note that as we're pulling from a monorepo (with multiple project in it)
 # we are using partial-clone and sparse-checkout to get the agama code

diff --git a/docker-jans-config-api/Dockerfile b/docker-jans-config-api/Dockerfile
@@ -43,7 +43,7 @@ RUN wget -q https://maven.jans.io/maven/io/jans/jython-installer/${JYTHON_VERSIO
 # ==========
 
 ENV CN_VERSION=1.0.6-SNAPSHOT
-ENV CN_BUILD_DATE='2022-12-25 08:17'
+ENV CN_BUILD_DATE='2022-12-29 11:35'
 ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-config-api-server/${CN_VERSION}/jans-config-api-server-${CN_VERSION}.war
 
 # Install Jans Config API
@@ -110,7 +110,7 @@ COPY conf/prometheus-config.yaml /opt/prometheus/
 # jans-linux-setup sync
 # =====================
 
-ENV JANS_SOURCE_VERSION=bd2cdf8501d60959498078bbb31650965c321c73
+ENV JANS_SOURCE_VERSION=d95fc492ca00ccee0ecc88132376c128203faefb
 ARG JANS_SETUP_DIR=jans-linux-setup/jans_setup
 ARG JANS_CONFIG_API_RESOURCES=jans-config-api/server/src/main/resources
 

diff --git a/docker-jans-fido2/Dockerfile b/docker-jans-fido2/Dockerfile
@@ -35,7 +35,7 @@ EXPOSE 8080
 # =====
 
 ENV CN_VERSION=1.0.6-SNAPSHOT
-ENV CN_BUILD_DATE='2022-12-25 08:15'
+ENV CN_BUILD_DATE='2022-12-29 08:50'
 ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-fido2-server/${CN_VERSION}/jans-fido2-server-${CN_VERSION}.war
 
 # Install FIDO2
@@ -54,7 +54,7 @@ RUN mkdir -p ${JETTY_BASE}/jans-fido2/webapps \
 # jans-linux-setup sync
 # =====================
 
-ENV JANS_SOURCE_VERSION=bd2cdf8501d60959498078bbb31650965c321c73
+ENV JANS_SOURCE_VERSION=d95fc492ca00ccee0ecc88132376c128203faefb
 ARG JANS_SETUP_DIR=jans-linux-setup/jans_setup
 
 # note that as we're pulling from a monorepo (with multiple project in it)

diff --git a/docker-jans-monolith/Dockerfile b/docker-jans-monolith/Dockerfile
@@ -38,7 +38,7 @@ EXPOSE 443 8080 1636
 # jans-linux-setup
 # =====================
 
-ENV JANS_SOURCE_VERSION=bd2cdf8501d60959498078bbb31650965c321c73
+ENV JANS_SOURCE_VERSION=d95fc492ca00ccee0ecc88132376c128203faefb
 
 # cleanup
 RUN rm -rf /tmp/jans

diff --git a/docker-jans-persistence-loader/Dockerfile b/docker-jans-persistence-loader/Dockerfile
@@ -24,7 +24,7 @@ RUN python3 -m ensurepip \
 # =====================
 
 # janssenproject/jans SHA commit
-ENV JANS_SOURCE_VERSION=bd2cdf8501d60959498078bbb31650965c321c73
+ENV JANS_SOURCE_VERSION=d95fc492ca00ccee0ecc88132376c128203faefb
 ARG JANS_SETUP_DIR=jans-linux-setup/jans_setup
 ARG JANS_SCRIPT_CATALOG_DIR=docs/script-catalog
 ARG JANS_CONFIG_API_RESOURCES=jans-config-api/server/src/main/resources

diff --git a/docker-jans-persistence-loader/scripts/spanner_setup.py b/docker-jans-persistence-loader/scripts/spanner_setup.py
@@ -421,6 +421,10 @@ def column_int_to_string(table_name, col_name):
         for mod in [
             ("jansPerson", "jansMobileDevices"),
             ("jansPerson", "jansOTPDevices"),
+            ("jansToken", "clnId"),
+            ("jansUmaRPT", "clnId"),
+            ("jansUmaPCT", "clnId"),
+            ("jansCibaReq", "clnId"),
         ]:
             column_from_array(mod[0], mod[1])
 

diff --git a/docker-jans-persistence-loader/scripts/sql_setup.py b/docker-jans-persistence-loader/scripts/sql_setup.py
@@ -5,6 +5,7 @@
 from string import Template
 from pathlib import Path
 
+from sqlalchemy import text
 from sqlalchemy.exc import OperationalError
 
 from jans.pycloudlib.persistence.sql import SqlClient
@@ -322,9 +323,28 @@ def column_from_json(table_name, col_name):
                 for row in self.client.search(table_name, ["doc_id", col_name])
             }
 
-            # to change the storage format of a JSON column, drop the column and
-            # add the column back specifying the new storage format
             with self.client.engine.connect() as conn:
+                # mysql will raise error if dropping column which has functional index,
+                # hence the associated index must be dropped first
+                if self.client.dialect == "mysql":
+                    for idx in conn.execute(
+                        text(
+                            "SELECT index_name "
+                            "FROM information_schema.statistics "
+                            "WHERE table_name = :table_name "
+                            "AND index_name LIKE :index_name '%' "
+                            "AND expression LIKE '%' :col_name '%';"
+                        ),
+                        {
+                            "table_name": table_name,
+                            "index_name": f"{table_name}_json_",
+                            "col_name": col_name
+                        },
+                    ):
+                        conn.execute(f"ALTER TABLE {table_name} DROP INDEX {idx[0]}")
+
+                # to change the storage format of a JSON column, drop the column and
+                # add the column back specifying the new storage format
                 conn.execute(f"ALTER TABLE {self.client.quoted_id(table_name)} DROP COLUMN {self.client.quoted_id(col_name)}")
                 conn.execute(f"ALTER TABLE {self.client.quoted_id(table_name)} ADD COLUMN {self.client.quoted_id(col_name)} {data_type}")
 
@@ -333,8 +353,10 @@ def column_from_json(table_name, col_name):
 
             # pre-populate the modified column
             for doc_id, value in values.items():
-                if value and "v" in value and value["v"]:
+                if self.client.dialect == "mysql" and value and value.get("v", []):
                     new_value = value["v"][0]
+                elif self.client.dialect == "pgsql" and value:
+                    new_value = value[0]
                 else:
                     new_value = ""
                 self.client.update(table_name, doc_id, {col_name: new_value})
@@ -427,6 +449,10 @@ def column_from_json(table_name, col_name):
         for mod in [
             ("jansPerson", "jansMobileDevices"),
             ("jansPerson", "jansOTPDevices"),
+            ("jansToken", "clnId"),
+            ("jansUmaRPT", "clnId"),
+            ("jansUmaPCT", "clnId"),
+            ("jansCibaReq", "clnId"),
         ]:
             column_from_json(mod[0], mod[1])
 

diff --git a/docker-jans-scim/Dockerfile b/docker-jans-scim/Dockerfile
@@ -46,7 +46,7 @@ RUN wget -q https://maven.jans.io/maven/io/jans/jython-installer/${JYTHON_VERSIO
 # ====
 
 ENV CN_VERSION=1.0.6-SNAPSHOT
-ENV CN_BUILD_DATE='2022-12-25 08:16'
+ENV CN_BUILD_DATE='2022-12-29 11:33'
 ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-scim-server/${CN_VERSION}/jans-scim-server-${CN_VERSION}.war
 
 # Install SCIM
@@ -80,7 +80,7 @@ COPY conf/prometheus-config.yaml /opt/prometheus/
 # jans-linux-setup sync
 # =====================
 
-ENV JANS_SOURCE_VERSION=bd2cdf8501d60959498078bbb31650965c321c73
+ENV JANS_SOURCE_VERSION=d95fc492ca00ccee0ecc88132376c128203faefb
 ARG JANS_SETUP_DIR=jans-linux-setup/jans_setup
 ARG JANS_SCIM_RESOURCE_DIR=jans-scim/server/src/main/resources
 

diff --git a/jans-pycloudlib/jans/pycloudlib/persistence/sql.py b/jans-pycloudlib/jans/pycloudlib/persistence/sql.py
@@ -258,6 +258,7 @@ def __init__(self, manager: Manager, *args: _t.Any, **kwargs: _t.Any) -> None:
             self.adapter = MysqlAdapter()
 
         self.dialect = self.adapter.dialect
+        self._metadata: _t.Optional[MetaData] = None
 
     @cached_property
     def engine(self) -> Engine:
@@ -274,24 +275,25 @@ def engine_url(self) -> str:
         password = get_sql_password(self.manager)
         return f"{self.adapter.connector}://{user}:{password}@{host}:{port}/{database}"
 
-    @cached_property
+    @property
     def metadata(self) -> MetaData:
         """Lazy init of metadata."""
-        with warnings.catch_warnings():
-            # postgresql driver will show warnings about unsupported reflection
-            # on expression-based index, i.e. `lower(uid::text)`; but we don't
-            # want to clutter the logs with these warnings, hence we suppress the
-            # warnings
-            warnings.filterwarnings(
-                "ignore",
-                message="Skipped unsupported reflection of expression-based index",
-                category=SAWarning,
-            )
-
-            # do reflection on database table
-            metadata = MetaData(bind=self.engine)
-            metadata.reflect()
-        return metadata
+        if not self._metadata:
+            with warnings.catch_warnings():
+                # postgresql driver will show warnings about unsupported reflection
+                # on expression-based index, i.e. `lower(uid::text)`; but we don't
+                # want to clutter the logs with these warnings, hence we suppress the
+                # warnings
+                warnings.filterwarnings(
+                    "ignore",
+                    message="Skipped unsupported reflection of expression-based index",
+                    category=SAWarning,
+                )
+
+                # do reflection on database table
+                self._metadata = MetaData(bind=self.engine)
+                self._metadata.reflect()
+        return self._metadata
 
     def connected(self) -> bool:
         """Check whether connection is alive by executing simple query."""