From dc87dc01c4c63d6fcc2b967ce97a52880083b95f Mon Sep 17 00:00:00 2001 From: Isman Firmansyah Date: Tue, 29 Mar 2022 14:54:58 +0700 Subject: [PATCH] fix: typo and indexing error (#1125) * fix: rename config httpLoggingExludePaths to httpLoggingExcludePaths * fix: skip unsupported/invalid indexes * fix: index creation on spanner persistence --- docker-jans-auth-server/Dockerfile | 2 +- docker-jans-config-api/Dockerfile | 2 +- docker-jans-persistence-loader/Dockerfile | 2 +- .../scripts/spanner_setup.py | 56 +------------------ .../scripts/sql_setup.py | 30 ++++------ .../scripts/upgrade.py | 4 ++ 6 files changed, 21 insertions(+), 75 deletions(-) diff --git a/docker-jans-auth-server/Dockerfile b/docker-jans-auth-server/Dockerfile index fbc2a6d2d7b..641acdbcb2c 100644 --- a/docker-jans-auth-server/Dockerfile +++ b/docker-jans-auth-server/Dockerfile @@ -53,7 +53,7 @@ RUN wget -q https://github.com/fabioz/PyDev.Debugger/archive/refs/tags/pydev_deb # =========== ENV CN_VERSION=1.0.0-SNAPSHOT -ENV CN_BUILD_DATE='2022-03-23 17:17' +ENV CN_BUILD_DATE='2022-03-28 08:56' ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-auth-server/${CN_VERSION}/jans-auth-server-${CN_VERSION}.war # Install Jans Auth diff --git a/docker-jans-config-api/Dockerfile b/docker-jans-config-api/Dockerfile index 9d185f258ec..8c6d6306fa1 100644 --- a/docker-jans-config-api/Dockerfile +++ b/docker-jans-config-api/Dockerfile @@ -33,7 +33,7 @@ RUN wget -q https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-home/${JETTY_ # ========== ENV CN_VERSION=1.0.0-SNAPSHOT -ENV CN_BUILD_DATE='2022-03-23 17:23' +ENV CN_BUILD_DATE='2022-03-28 10:22' ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-config-api-server/${CN_VERSION}/jans-config-api-server-${CN_VERSION}.war # Install Jans Config API diff --git a/docker-jans-persistence-loader/Dockerfile b/docker-jans-persistence-loader/Dockerfile index 3a9a2d51ff7..743fe498d71 100644 --- a/docker-jans-persistence-loader/Dockerfile +++ b/docker-jans-persistence-loader/Dockerfile @@ -22,7 +22,7 @@ RUN pip3 install -U pip wheel \ # jans-linux-setup sync # ===================== -ENV JANS_LINUX_SETUP_VERSION=e3d9dbffdab29d58d31dab004f5d392f5ada0591 +ENV JANS_LINUX_SETUP_VERSION=41b6fa185505d3a6a1b5423a6f6df38337a14168 ARG JANS_SETUP_DIR=jans-linux-setup/jans_setup # note that as we're pulling from a monorepo (with multiple project in it) diff --git a/docker-jans-persistence-loader/scripts/spanner_setup.py b/docker-jans-persistence-loader/scripts/spanner_setup.py index 18889bf40e3..ff42c62cc8a 100644 --- a/docker-jans-persistence-loader/scripts/spanner_setup.py +++ b/docker-jans-persistence-loader/scripts/spanner_setup.py @@ -1,4 +1,3 @@ -# import itertools import hashlib import json import logging.config @@ -54,11 +53,6 @@ def __init__(self, manager): with open(f"/app/static/rdbm/{index_fn}") as f: self.sql_indexes = json.loads(f.read()) - # with open("/app/static/couchbase/index.json") as f: - # # prefix = os.environ.get("CN_COUCHBASE_BUCKET_PREFIX", "jans") - # txt = f.read() # .replace("!bucket_prefix!", prefix) - # self.cb_indexes = json.loads(txt) - with open("/app/static/rdbm/sub_tables.json") as f: self.sub_tables = json.loads(f.read()).get(self.db_dialect) or {} @@ -93,31 +87,15 @@ def get_data_type(self, attr, table=None): syntax_def = self.sql_data_types_mapping[syntax] type_ = syntax_def.get(self.db_dialect) # or syntax_def["mysql"] - # char_type = "VARCHAR" - # if self.db_dialect == "spanner": char_type = "STRING" if type_["type"] != char_type: # not STRING data_type = type_["type"] else: - # if "size" in type_: - # size = type_["size"] - # # data_type = f"{char_type}(type['size'])" - # else: - # # data_type = "STRING(MAX)" - # size = "MAX" size = type_.get("size") or "MAX" data_type = f"{char_type}({size})" - # if type_["size"] <= 127: - # data_type = f"{char_type}({type_['size']})" - # elif type_["size"] <= 255: - # data_type = "TINYTEXT" if self.db_dialect == "mysql" else "TEXT" - # else: - # data_type = "TEXT" - - # if data_type == "TEXT" and self.db_dialect == "spanner": - # data_type = "STRING(MAX)" + return data_type def create_tables(self): @@ -175,32 +153,9 @@ def create_tables(self): # sql_cmd = f"ALTER TABLE {table} ADD {col_def};" # logger.info(sql_cmd) - # def _fields_from_cb_indexes(self): - # fields = [] - - # for _, data in self.cb_indexes.items(): - # # extract and flatten - # attrs = list(itertools.chain.from_iterable(data["attributes"])) - # fields += attrs - - # for static in data["static"]: - # attrs = [ - # attr for attr in static[0] - # if "(" not in attr - # ] - # fields += attrs - - # fields = list(set(fields)) - # # exclude objectClass - # if "objectClass" in fields: - # fields.remove("objectClass") - # return fields - def get_index_fields(self, table_name): - # cb_fields = self._fields_from_cb_indexes() fields = self.sql_indexes.get(table_name, {}).get("fields", []) fields += self.sql_indexes["__common__"]["fields"] - # fields += cb_fields # make unique fields return list(set(fields)) @@ -214,7 +169,7 @@ def create_spanner_indexes(self, table_name: str, column_mapping: dict): index_name = f"{table_name}_{FIELD_RE.sub('_', column_name)}" - if column_type.lower() != "array": + if not column_type.lower().startswith("array"): query = f"CREATE INDEX {self.client.quoted_id(index_name)} ON {self.client.quoted_id(table_name)} ({self.client.quoted_id(column_name)})" self.client.create_index(query) else: @@ -279,12 +234,8 @@ def transform_value(self, key, values): if data_type in ("DATETIME(3)", "TIMESTAMP",): dval = values[0].strip("Z") - # sep = " " - # postfix = "" - # if self.db_dialect == "spanner": sep = "T" postfix = "Z" - # return "{}-{}-{} {}:{}:{}{}".format(dval[0:4], dval[4:6], dval[6:8], dval[8:10], dval[10:12], dval[12:14], dval[14:17]) return "{}-{}-{}{}{}:{}:{}{}{}".format( dval[0:4], dval[4:6], @@ -298,7 +249,6 @@ def transform_value(self, key, values): ) if data_type == "JSON": - # return json.dumps({"v": values}) return {"v": values} if data_type == "ARRAY": @@ -324,8 +274,6 @@ def data_from_ldif(self, filename): table_name = oc[-1] - # entry.pop(rdn_name) - if "objectClass" in entry: entry.pop("objectClass") elif "objectclass" in entry: diff --git a/docker-jans-persistence-loader/scripts/sql_setup.py b/docker-jans-persistence-loader/scripts/sql_setup.py index 03aa907e4de..da3fbdfe407 100644 --- a/docker-jans-persistence-loader/scripts/sql_setup.py +++ b/docker-jans-persistence-loader/scripts/sql_setup.py @@ -9,6 +9,8 @@ from pathlib import Path from ldif import LDIFParser +from sqlalchemy.exc import NotSupportedError +from sqlalchemy.exc import OperationalError from jans.pycloudlib.persistence.sql import SQLClient @@ -91,8 +93,6 @@ def get_data_type(self, attr, table=None): type_ = syntax_def.get(self.db_dialect) or syntax_def["mysql"] char_type = "VARCHAR" - if self.db_dialect == "spanner": - char_type = "STRING" if type_["type"] != char_type: data_type = type_["type"] @@ -104,8 +104,6 @@ def get_data_type(self, attr, table=None): else: data_type = "TEXT" - if data_type == "TEXT" and self.db_dialect == "spanner": - data_type = "STRING(MAX)" return data_type def create_tables(self): @@ -138,8 +136,8 @@ def create_tables(self): doc_id_type = self.get_data_type("doc_id", table) table_columns[table].update({ "doc_id": doc_id_type, - "objectClass": "VARCHAR(48)" if self.db_dialect != "spanner" else "STRING(48)", - "dn": "VARCHAR(128)" if self.db_dialect != "spanner" else "STRING(128)", + "objectClass": "VARCHAR(48)", + "dn": "VARCHAR(128)", }) # make sure ``oc["may"]`` doesn't have duplicate attribute @@ -177,9 +175,8 @@ def create_mysql_indexes(self, table_name: str, column_mapping: dict): if column_name == "doc_id" or column_name not in fields: continue - index_name = f"{table_name}_{FIELD_RE.sub('_', column_name)}" - if column_type.lower() != "json": + index_name = f"{table_name}_{FIELD_RE.sub('_', column_name)}" query = f"CREATE INDEX {self.client.quoted_id(index_name)} ON {self.client.quoted_id(table_name)} ({self.client.quoted_id(column_name)})" self.client.create_index(query) else: @@ -193,8 +190,13 @@ def create_mysql_indexes(self, table_name: str, column_mapping: dict): "field": column_name, "data_type": column_type, }) name = f"{table_name}_json_{i}" - query = f"CREATE INDEX {self.client.quoted_id(name)} ON {self.client.quoted_id(table_name)} (({index_str_fmt}))" - self.client.create_index(query) + query = f"ALTER TABLE {self.client.quoted_id(table_name)} ADD INDEX {self.client.quoted_id(name)} (({index_str_fmt}))" + + try: + self.client.create_index(query) + except (NotSupportedError, OperationalError) as exc: + msg = exc.orig.args[1] if self.db_dialect == "mysql" else exc.orig.pgerror + logger.warning(f"Failed to create index {name} for {table_name}.{column_name} column; reason={msg}") for i, custom in enumerate(self.sql_indexes.get(table_name, {}).get("custom", []), start=1): # jansPerson table has unsupported custom index expressions that need to be skipped if mysql < 8.0 @@ -290,10 +292,6 @@ def transform_value(self, key, values): dval = values[0].strip("Z") sep = " " postfix = "" - if self.db_dialect == "spanner": - sep = "T" - postfix = "Z" - # return "{}-{}-{} {}:{}:{}{}".format(dval[0:4], dval[4:6], dval[6:8], dval[8:10], dval[10:12], dval[12:14], dval[14:17]) return "{}-{}-{}{}{}:{}:{}{}{}".format( dval[0:4], dval[4:6], @@ -307,12 +305,8 @@ def transform_value(self, key, values): ) if data_type == "JSON": - # return json.dumps({"v": values}) return {"v": values} - if data_type == "ARRAY": - return values - # fallback return values[0] diff --git a/docker-jans-persistence-loader/scripts/upgrade.py b/docker-jans-persistence-loader/scripts/upgrade.py index 8b411e30c23..c19025c9596 100644 --- a/docker-jans-persistence-loader/scripts/upgrade.py +++ b/docker-jans-persistence-loader/scripts/upgrade.py @@ -115,6 +115,10 @@ def _transform_auth_dynamic_config(conf): conf["useHighestLevelScriptIfAcrScriptNotFound"] = True should_update = True + if "httpLoggingExcludePaths" not in conf: + conf["httpLoggingExcludePaths"] = conf.pop("httpLoggingExludePaths", []) + should_update = True + # return the conf and flag to determine whether it needs update or not return conf, should_update