From decb84f34895872359cfd4c97ed075cc06bf797f Mon Sep 17 00:00:00 2001 From: A-Baji Date: Wed, 7 Jun 2023 15:42:29 +0000 Subject: [PATCH 01/16] alternate hidden attr implementation --- datajoint/fetch.py | 2 +- datajoint/heading.py | 19 ++++++++++++++----- datajoint/table.py | 4 +++- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/datajoint/fetch.py b/datajoint/fetch.py index 750939e5e..5ac9f8815 100644 --- a/datajoint/fetch.py +++ b/datajoint/fetch.py @@ -286,7 +286,7 @@ def __call__( ret = np.array(ret, dtype=record_type) except Exception as e: raise e - for name in heading: + for name in heading.names: # unpack blobs and externals ret[name] = list(map(partial(get, heading[name]), ret[name])) if format == "frame": diff --git a/datajoint/heading.py b/datajoint/heading.py index 9a782fc0e..2ced84d36 100644 --- a/datajoint/heading.py +++ b/datajoint/heading.py @@ -33,6 +33,7 @@ is_attachment=False, is_filepath=False, is_external=False, + is_hidden=False, adapter=None, store=None, unsupported=False, @@ -124,19 +125,21 @@ def attributes(self): @property def names(self): - return [k for k in self.attributes] + return [k for k in self.attributes if not self.attributes[k].is_hidden] @property def primary_key(self): - return [k for k, v in self.attributes.items() if v.in_key] + return [k for k, v in self.attributes.items() if v.in_key and not v.is_hidden] @property def secondary_attributes(self): - return [k for k, v in self.attributes.items() if not v.in_key] + return [ + k for k, v in self.attributes.items() if not v.in_key and not v.is_hidden + ] @property def blobs(self): - return [k for k, v in self.attributes.items() if v.is_blob] + return [k for k, v in self.attributes.items() if v.is_blob and not v.is_hidden] @property def non_blobs(self): @@ -144,12 +147,15 @@ def non_blobs(self): k for k, v in self.attributes.items() if not (v.is_blob or v.is_attachment or v.is_filepath or v.json) + and not v.is_hidden ] @property def new_attributes(self): return [ - k for k, v in self.attributes.items() if v.attribute_expression is not None + k + for k, v in self.attributes.items() + if v.attribute_expression is not None and not v.is_hidden ] def __getitem__(self, name): @@ -165,6 +171,8 @@ def __repr__(self): if self._table_status is not None: ret += "# " + self.table_status["comment"] + "\n" for v in self.attributes.values(): + if v.is_hidden: + continue if in_key and not v.in_key: ret += "---\n" in_key = False @@ -298,6 +306,7 @@ def _init_from_database(self): store=None, is_external=False, attribute_expression=None, + is_hidden=attr["name"].startswith("_"), ) if any(TYPE_PATTERN[t].match(attr["type"]) for t in ("INTEGER", "FLOAT")): diff --git a/datajoint/table.py b/datajoint/table.py index e047c8adc..57902014e 100644 --- a/datajoint/table.py +++ b/datajoint/table.py @@ -711,6 +711,8 @@ def describe(self, context=None, printout=False): attributes_declared = set() indexes = self.heading.indexes.copy() for attr in self.heading.attributes.values(): + if attr.is_hidden: + continue if in_key and not attr.in_key: definition += "---\n" in_key = False @@ -857,7 +859,7 @@ def check_fields(fields): raise KeyError( "`{0:s}` is not in the table heading".format(field) ) - elif set(field_list) != set(fields).intersection(self.heading.names): + elif set(field_list) != set(fields).intersection(self.heading): raise DataJointError("Attempt to insert rows with different fields.") if isinstance(row, np.void): # np.array From a49e2a3c590d86830942193ffbc465576e2289fa Mon Sep 17 00:00:00 2001 From: A-Baji Date: Wed, 7 Jun 2023 15:54:40 +0000 Subject: [PATCH 02/16] exclude from user queries --- datajoint/heading.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datajoint/heading.py b/datajoint/heading.py index 2ced84d36..649443bb8 100644 --- a/datajoint/heading.py +++ b/datajoint/heading.py @@ -206,6 +206,7 @@ def as_sql(self, fields, include_aliases=True): else self.attributes[name].attribute_expression + (" as `%s`" % name if include_aliases else "") for name in fields + if not self.attributes[name].is_hidden ) def __iter__(self): From aa96fe382da8d409b860552f03c490c26f3e592d Mon Sep 17 00:00:00 2001 From: A-Baji Date: Fri, 9 Jun 2023 17:34:03 +0000 Subject: [PATCH 03/16] timestamp metadata implementation --- datajoint/declare.py | 24 ++++++++++++++++++------ datajoint/heading.py | 20 ++++++-------------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/datajoint/declare.py b/datajoint/declare.py index 683e34759..ae8641c74 100644 --- a/datajoint/declare.py +++ b/datajoint/declare.py @@ -114,12 +114,12 @@ def build_foreign_key_parser(): return arrow + options + ref_table -def build_attribute_parser(): +def build_attribute_parser(parse_metadata=False): quoted = pp.QuotedString('"') ^ pp.QuotedString("'") colon = pp.Literal(":").suppress() - attribute_name = pp.Word(pp.srange("[a-z]"), pp.srange("[a-z0-9_]")).setResultsName( - "name" - ) + attribute_name = pp.Word( + pp.srange(f"[a-z{'_' if parse_metadata else ''}]"), pp.srange("[a-z0-9_]") + ).setResultsName("name") data_type = ( pp.Combine(pp.Word(pp.alphas) + pp.SkipTo("#", ignore=quoted)) ^ pp.QuotedString("<", endQuoteChar=">", unquoteResults=False) @@ -134,6 +134,7 @@ def build_attribute_parser(): foreign_key_parser_old = build_foreign_key_parser_old() foreign_key_parser = build_foreign_key_parser() attribute_parser = build_attribute_parser() +metadata_attribute_parser = build_attribute_parser(parse_metadata=True) def is_foreign_key(line): @@ -245,6 +246,7 @@ def prepare_declare(definition, context): foreign_key_sql = [] index_sql = [] external_stores = [] + metadata_attributes = ["_timestamp = CURRENT_TIMESTAMP : timestamp"] for line in definition: if not line or line.startswith("#"): # ignore additional comments @@ -272,6 +274,12 @@ def prepare_declare(definition, context): if name not in attributes: attributes.append(name) attribute_sql.append(sql) + for line in metadata_attributes: + name, sql, store = compile_attribute( + line, in_key, foreign_key_sql, context, is_metadata=True + ) + attributes.append(name) + attribute_sql.append(sql) return ( table_comment, @@ -496,7 +504,7 @@ def substitute_special_type(match, category, foreign_key_sql, context): assert False, "Unknown special type" -def compile_attribute(line, in_key, foreign_key_sql, context): +def compile_attribute(line, in_key, foreign_key_sql, context, is_metadata=False): """ Convert attribute definition from DataJoint format to SQL @@ -504,10 +512,14 @@ def compile_attribute(line, in_key, foreign_key_sql, context): :param in_key: set to True if attribute is in primary key set :param foreign_key_sql: the list of foreign key declarations to add to :param context: context in which to look up user-defined attribute type adapterss + :param is_metadata: flag to use an alternate parser for metadata attributes :returns: (name, sql, is_external) -- attribute name and sql code for its declaration """ try: - match = attribute_parser.parseString(line + "#", parseAll=True) + if is_metadata: + match = metadata_attribute_parser.parseString(line + "#", parseAll=True) + else: + match = attribute_parser.parseString(line + "#", parseAll=True) except pp.ParseException as err: raise DataJointError( "Declaration error in position {pos} in line:\n {line}\n{msg}".format( diff --git a/datajoint/heading.py b/datajoint/heading.py index 649443bb8..c2d8ed527 100644 --- a/datajoint/heading.py +++ b/datajoint/heading.py @@ -121,25 +121,23 @@ def table_status(self): def attributes(self): if self._attributes is None: self._init_from_database() # lazy loading from database - return self._attributes + return {k: v for k, v in self._attributes.items() if not v.is_hidden} @property def names(self): - return [k for k in self.attributes if not self.attributes[k].is_hidden] + return [k for k in self.attributes] @property def primary_key(self): - return [k for k, v in self.attributes.items() if v.in_key and not v.is_hidden] + return [k for k, v in self.attributes.items() if v.in_key] @property def secondary_attributes(self): - return [ - k for k, v in self.attributes.items() if not v.in_key and not v.is_hidden - ] + return [k for k, v in self.attributes.items() if not v.in_key] @property def blobs(self): - return [k for k, v in self.attributes.items() if v.is_blob and not v.is_hidden] + return [k for k, v in self.attributes.items() if v.is_blob] @property def non_blobs(self): @@ -147,15 +145,12 @@ def non_blobs(self): k for k, v in self.attributes.items() if not (v.is_blob or v.is_attachment or v.is_filepath or v.json) - and not v.is_hidden ] @property def new_attributes(self): return [ - k - for k, v in self.attributes.items() - if v.attribute_expression is not None and not v.is_hidden + k for k, v in self.attributes.items() if v.attribute_expression is not None ] def __getitem__(self, name): @@ -171,8 +166,6 @@ def __repr__(self): if self._table_status is not None: ret += "# " + self.table_status["comment"] + "\n" for v in self.attributes.values(): - if v.is_hidden: - continue if in_key and not v.in_key: ret += "---\n" in_key = False @@ -206,7 +199,6 @@ def as_sql(self, fields, include_aliases=True): else self.attributes[name].attribute_expression + (" as `%s`" % name if include_aliases else "") for name in fields - if not self.attributes[name].is_hidden ) def __iter__(self): From b119768e31e6bcfd178b9e644d158ba739177457 Mon Sep 17 00:00:00 2001 From: A-Baji Date: Fri, 9 Jun 2023 17:48:16 +0000 Subject: [PATCH 04/16] revert --- datajoint/fetch.py | 2 +- datajoint/table.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datajoint/fetch.py b/datajoint/fetch.py index 5ac9f8815..750939e5e 100644 --- a/datajoint/fetch.py +++ b/datajoint/fetch.py @@ -286,7 +286,7 @@ def __call__( ret = np.array(ret, dtype=record_type) except Exception as e: raise e - for name in heading.names: + for name in heading: # unpack blobs and externals ret[name] = list(map(partial(get, heading[name]), ret[name])) if format == "frame": diff --git a/datajoint/table.py b/datajoint/table.py index 57902014e..21975f6b5 100644 --- a/datajoint/table.py +++ b/datajoint/table.py @@ -859,7 +859,7 @@ def check_fields(fields): raise KeyError( "`{0:s}` is not in the table heading".format(field) ) - elif set(field_list) != set(fields).intersection(self.heading): + elif set(field_list) != set(fields).intersection(self.heading.names): raise DataJointError("Attempt to insert rows with different fields.") if isinstance(row, np.void): # np.array From 27ba8b14fa3cc15ed554fb953213a481525590b2 Mon Sep 17 00:00:00 2001 From: A-Baji Date: Mon, 12 Jun 2023 15:51:00 +0000 Subject: [PATCH 05/16] insert blobs sql fix --- tests_old/test_blob_matlab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_old/test_blob_matlab.py b/tests_old/test_blob_matlab.py index 6104c9291..a2fa67fd8 100644 --- a/tests_old/test_blob_matlab.py +++ b/tests_old/test_blob_matlab.py @@ -40,7 +40,7 @@ def insert_blobs(): schema.connection.query( """ - INSERT INTO {table_name} VALUES + INSERT INTO {table_name} (`id`, `comment`, `blob`) VALUES (1,'simple string',0x6D596D00410200000000000000010000000000000010000000000000000400000000000000630068006100720061006300740065007200200073007400720069006E006700), (2,'1D vector',0x6D596D0041020000000000000001000000000000000C000000000000000600000000000000000000000000F03F00000000000030400000000000003F4000000000000047400000000000804E4000000000000053400000000000C056400000000000805A400000000000405E4000000000000061400000000000E062400000000000C06440), (3,'string array',0x6D596D00430200000000000000010000000000000002000000000000002F0000000000000041020000000000000001000000000000000700000000000000040000000000000073007400720069006E00670031002F0000000000000041020000000000000001000000000000000700000000000000040000000000000073007400720069006E0067003200), From c046870acf0baff3316ff20d560c9cc51476d04e Mon Sep 17 00:00:00 2001 From: A-Baji Date: Mon, 12 Jun 2023 19:06:21 +0000 Subject: [PATCH 06/16] bump nginx --- LNX-docker-compose.yml | 2 +- local-docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LNX-docker-compose.yml b/LNX-docker-compose.yml index 970552860..9c0a95b78 100644 --- a/LNX-docker-compose.yml +++ b/LNX-docker-compose.yml @@ -44,7 +44,7 @@ services: interval: 15s fakeservices.datajoint.io: <<: *net - image: datajoint/nginx:v0.2.5 + image: datajoint/nginx:v0.2.6 environment: - ADD_db_TYPE=DATABASE - ADD_db_ENDPOINT=db:3306 diff --git a/local-docker-compose.yml b/local-docker-compose.yml index 8b43289d3..62b52ad66 100644 --- a/local-docker-compose.yml +++ b/local-docker-compose.yml @@ -46,7 +46,7 @@ services: interval: 15s fakeservices.datajoint.io: <<: *net - image: datajoint/nginx:v0.2.5 + image: datajoint/nginx:v0.2.6 environment: - ADD_db_TYPE=DATABASE - ADD_db_ENDPOINT=db:3306 From 611da2f9775345ca62a22c2556aef444db4501c6 Mon Sep 17 00:00:00 2001 From: A-Baji Date: Tue, 13 Jun 2023 17:15:59 +0000 Subject: [PATCH 07/16] simplify --- datajoint/declare.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/datajoint/declare.py b/datajoint/declare.py index ae8641c74..144683598 100644 --- a/datajoint/declare.py +++ b/datajoint/declare.py @@ -114,12 +114,12 @@ def build_foreign_key_parser(): return arrow + options + ref_table -def build_attribute_parser(parse_metadata=False): +def build_attribute_parser(): quoted = pp.QuotedString('"') ^ pp.QuotedString("'") colon = pp.Literal(":").suppress() - attribute_name = pp.Word( - pp.srange(f"[a-z{'_' if parse_metadata else ''}]"), pp.srange("[a-z0-9_]") - ).setResultsName("name") + attribute_name = pp.Word(pp.srange("[a-z]"), pp.srange("[a-z0-9_]")).setResultsName( + "name" + ) data_type = ( pp.Combine(pp.Word(pp.alphas) + pp.SkipTo("#", ignore=quoted)) ^ pp.QuotedString("<", endQuoteChar=">", unquoteResults=False) @@ -134,7 +134,6 @@ def build_attribute_parser(parse_metadata=False): foreign_key_parser_old = build_foreign_key_parser_old() foreign_key_parser = build_foreign_key_parser() attribute_parser = build_attribute_parser() -metadata_attribute_parser = build_attribute_parser(parse_metadata=True) def is_foreign_key(line): @@ -246,7 +245,6 @@ def prepare_declare(definition, context): foreign_key_sql = [] index_sql = [] external_stores = [] - metadata_attributes = ["_timestamp = CURRENT_TIMESTAMP : timestamp"] for line in definition: if not line or line.startswith("#"): # ignore additional comments @@ -274,12 +272,6 @@ def prepare_declare(definition, context): if name not in attributes: attributes.append(name) attribute_sql.append(sql) - for line in metadata_attributes: - name, sql, store = compile_attribute( - line, in_key, foreign_key_sql, context, is_metadata=True - ) - attributes.append(name) - attribute_sql.append(sql) return ( table_comment, @@ -316,6 +308,7 @@ def declare(full_table_name, definition, context): index_sql, external_stores, ) = prepare_declare(definition, context) + attribute_sql.extend(["`_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP"]) if not primary_key: raise DataJointError("Table must have a primary key") @@ -504,7 +497,7 @@ def substitute_special_type(match, category, foreign_key_sql, context): assert False, "Unknown special type" -def compile_attribute(line, in_key, foreign_key_sql, context, is_metadata=False): +def compile_attribute(line, in_key, foreign_key_sql, context): """ Convert attribute definition from DataJoint format to SQL @@ -512,14 +505,10 @@ def compile_attribute(line, in_key, foreign_key_sql, context, is_metadata=False) :param in_key: set to True if attribute is in primary key set :param foreign_key_sql: the list of foreign key declarations to add to :param context: context in which to look up user-defined attribute type adapterss - :param is_metadata: flag to use an alternate parser for metadata attributes :returns: (name, sql, is_external) -- attribute name and sql code for its declaration """ try: - if is_metadata: - match = metadata_attribute_parser.parseString(line + "#", parseAll=True) - else: - match = attribute_parser.parseString(line + "#", parseAll=True) + match = attribute_parser.parseString(line + "#", parseAll=True) except pp.ParseException as err: raise DataJointError( "Declaration error in position {pos} in line:\n {line}\n{msg}".format( From 59f0315c435b4ac2c2190ca711f742fbf253e580 Mon Sep 17 00:00:00 2001 From: A-Baji Date: Tue, 13 Jun 2023 17:17:48 +0000 Subject: [PATCH 08/16] unnecessary --- datajoint/table.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/datajoint/table.py b/datajoint/table.py index 21975f6b5..e047c8adc 100644 --- a/datajoint/table.py +++ b/datajoint/table.py @@ -711,8 +711,6 @@ def describe(self, context=None, printout=False): attributes_declared = set() indexes = self.heading.indexes.copy() for attr in self.heading.attributes.values(): - if attr.is_hidden: - continue if in_key and not attr.in_key: definition += "---\n" in_key = False From db41ecbbc873b2132c3287903c700a6dfca17725 Mon Sep 17 00:00:00 2001 From: A-Baji Date: Tue, 13 Jun 2023 19:49:56 +0000 Subject: [PATCH 09/16] alter declare --- datajoint/declare.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/datajoint/declare.py b/datajoint/declare.py index 144683598..73a503499 100644 --- a/datajoint/declare.py +++ b/datajoint/declare.py @@ -16,6 +16,7 @@ "NULL", } # SQL literals to be used without quotes (case insensitive) EXTERNAL_TABLE_ROOT = "~external" +METADATA_ATTRIBUTES_SQL = ["`_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP"] TYPE_PATTERN = { k: re.compile(v, re.I) @@ -308,7 +309,7 @@ def declare(full_table_name, definition, context): index_sql, external_stores, ) = prepare_declare(definition, context) - attribute_sql.extend(["`_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP"]) + attribute_sql.extend(METADATA_ATTRIBUTES_SQL) if not primary_key: raise DataJointError("Table must have a primary key") @@ -411,6 +412,7 @@ def alter(definition, old_definition, context): index_sql, external_stores, ) = prepare_declare(definition, context) + attribute_sql.extend(METADATA_ATTRIBUTES_SQL) ( table_comment_, primary_key_, @@ -419,6 +421,7 @@ def alter(definition, old_definition, context): index_sql_, external_stores_, ) = prepare_declare(old_definition, context) + attribute_sql_.extend(METADATA_ATTRIBUTES_SQL) # analyze differences between declarations sql = list() From 0d89af2f45b62b8ebc1796712d4c2ff41f138892 Mon Sep 17 00:00:00 2001 From: A-Baji Date: Wed, 14 Jun 2023 19:56:42 +0000 Subject: [PATCH 10/16] unqiue metadata attr name per table --- datajoint/declare.py | 23 ++++++++++++++++------- tests_old/test_declare.py | 15 ++++++++++++++- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/datajoint/declare.py b/datajoint/declare.py index 73a503499..5470e5418 100644 --- a/datajoint/declare.py +++ b/datajoint/declare.py @@ -16,7 +16,10 @@ "NULL", } # SQL literals to be used without quotes (case insensitive) EXTERNAL_TABLE_ROOT = "~external" -METADATA_ATTRIBUTES_SQL = ["`_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP"] +METADATA_ATTRIBUTES_SQL = [ + "`_{full_table_name}_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP" +] +LONGEST_METADATA_ATTRIBUTE = len("__timestamp") TYPE_PATTERN = { k: re.compile(v, re.I) @@ -293,11 +296,14 @@ def declare(full_table_name, definition, context): :param context: dictionary of objects that might be referred to in the table :return: SQL CREATE TABLE statement, list of external stores used """ - table_name = full_table_name.strip("`").split(".")[1] - if len(table_name) > MAX_TABLE_NAME_LENGTH: + if ( + len(full_table_name.replace("`", "")) + LONGEST_METADATA_ATTRIBUTE + > MAX_TABLE_NAME_LENGTH + ): raise DataJointError( "Table name `{name}` exceeds the max length of {max_length}".format( - name=table_name, max_length=MAX_TABLE_NAME_LENGTH + name=full_table_name.replace("`", ""), + max_length=MAX_TABLE_NAME_LENGTH - LONGEST_METADATA_ATTRIBUTE, ) ) @@ -309,7 +315,12 @@ def declare(full_table_name, definition, context): index_sql, external_stores, ) = prepare_declare(definition, context) - attribute_sql.extend(METADATA_ATTRIBUTES_SQL) + attribute_sql.extend( + [ + attr.format(full_table_name=full_table_name.replace("`", "")) + for attr in METADATA_ATTRIBUTES_SQL + ] + ) if not primary_key: raise DataJointError("Table must have a primary key") @@ -412,7 +423,6 @@ def alter(definition, old_definition, context): index_sql, external_stores, ) = prepare_declare(definition, context) - attribute_sql.extend(METADATA_ATTRIBUTES_SQL) ( table_comment_, primary_key_, @@ -421,7 +431,6 @@ def alter(definition, old_definition, context): index_sql_, external_stores_, ) = prepare_declare(old_definition, context) - attribute_sql_.extend(METADATA_ATTRIBUTES_SQL) # analyze differences between declarations sql = list() diff --git a/tests_old/test_declare.py b/tests_old/test_declare.py index 67f532449..dcbb82485 100644 --- a/tests_old/test_declare.py +++ b/tests_old/test_declare.py @@ -6,6 +6,8 @@ raises, assert_set_equal, ) + +from tests_old.schema_simple import IJ, JI from .schema import * import datajoint as dj import inspect @@ -332,7 +334,7 @@ def test_long_table_name(): """ @schema - class WhyWouldAnyoneCreateATableNameThisLong(dj.Manual): + class WhyWouldAnyoneCreateATableNameThis(dj.Manual): definition = """ master : int """ @@ -341,3 +343,14 @@ class WithSuchALongPartNameThatItCrashesMySQL(dj.Part): definition = """ -> (master) """ + + @staticmethod + def test_hidden_attributes(): + assert ( + list(JI().heading._attributes.keys())[-1] + == "_djtest_relational.#j_i_timestamp" + ) + assert ( + list(JI().heading.attributes.keys())[-1] + != "_djtest_relational.#j_i_timestamp" + ) From 5649c98e8a40a94cea4479d0a0a27cdc5f93a576 Mon Sep 17 00:00:00 2001 From: A-Baji Date: Fri, 16 Jun 2023 16:46:25 +0000 Subject: [PATCH 11/16] shorten attr name if too long --- datajoint/declare.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/datajoint/declare.py b/datajoint/declare.py index 5470e5418..0ae2ff9cd 100644 --- a/datajoint/declare.py +++ b/datajoint/declare.py @@ -296,14 +296,11 @@ def declare(full_table_name, definition, context): :param context: dictionary of objects that might be referred to in the table :return: SQL CREATE TABLE statement, list of external stores used """ - if ( - len(full_table_name.replace("`", "")) + LONGEST_METADATA_ATTRIBUTE - > MAX_TABLE_NAME_LENGTH - ): + table_name = full_table_name.strip("`").split(".")[1] + if len(table_name) > MAX_TABLE_NAME_LENGTH: raise DataJointError( "Table name `{name}` exceeds the max length of {max_length}".format( - name=full_table_name.replace("`", ""), - max_length=MAX_TABLE_NAME_LENGTH - LONGEST_METADATA_ATTRIBUTE, + name=table_name, max_length=MAX_TABLE_NAME_LENGTH ) ) @@ -317,7 +314,11 @@ def declare(full_table_name, definition, context): ) = prepare_declare(definition, context) attribute_sql.extend( [ - attr.format(full_table_name=full_table_name.replace("`", "")) + attr.format( + full_table_name=full_table_name.replace("`", "")[ + 0 : 64 - LONGEST_METADATA_ATTRIBUTE + ] + ) for attr in METADATA_ATTRIBUTES_SQL ] ) From 316237e6786f2131a96e465565fd38e9255de548 Mon Sep 17 00:00:00 2001 From: A-Baji Date: Fri, 16 Jun 2023 16:48:21 +0000 Subject: [PATCH 12/16] fix test table name --- tests_old/test_declare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_old/test_declare.py b/tests_old/test_declare.py index dcbb82485..5f71135d3 100644 --- a/tests_old/test_declare.py +++ b/tests_old/test_declare.py @@ -334,7 +334,7 @@ def test_long_table_name(): """ @schema - class WhyWouldAnyoneCreateATableNameThis(dj.Manual): + class WhyWouldAnyoneCreateATableNameThisLong(dj.Manual): definition = """ master : int """ From a2e464df308f2666ddf677ab7263519a1b745c6c Mon Sep 17 00:00:00 2001 From: A-Baji Date: Fri, 16 Jun 2023 16:54:15 +0000 Subject: [PATCH 13/16] clean up test --- tests_old/test_declare.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests_old/test_declare.py b/tests_old/test_declare.py index 5f71135d3..2a465eb58 100644 --- a/tests_old/test_declare.py +++ b/tests_old/test_declare.py @@ -7,7 +7,6 @@ assert_set_equal, ) -from tests_old.schema_simple import IJ, JI from .schema import * import datajoint as dj import inspect @@ -347,10 +346,10 @@ class WithSuchALongPartNameThatItCrashesMySQL(dj.Part): @staticmethod def test_hidden_attributes(): assert ( - list(JI().heading._attributes.keys())[-1] - == "_djtest_relational.#j_i_timestamp" + list(Experiment().heading._attributes.keys())[-1] + == "_djtest_test1._experiment_timestamp" ) assert ( - list(JI().heading.attributes.keys())[-1] - != "_djtest_relational.#j_i_timestamp" + list(Experiment().heading.attributes.keys())[-1] + != "_djtest_test1._experiment_timestamp" ) From b1019f45603e2ac14c9f3504d5b4a9e201f78308 Mon Sep 17 00:00:00 2001 From: A-Baji Date: Fri, 16 Jun 2023 16:55:52 +0000 Subject: [PATCH 14/16] whiteline --- tests_old/test_declare.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests_old/test_declare.py b/tests_old/test_declare.py index 2a465eb58..4a223c649 100644 --- a/tests_old/test_declare.py +++ b/tests_old/test_declare.py @@ -6,7 +6,6 @@ raises, assert_set_equal, ) - from .schema import * import datajoint as dj import inspect From 717d4b5179302658d2b7e9dad4d650e7b0cd4589 Mon Sep 17 00:00:00 2001 From: A-Baji Date: Fri, 23 Jun 2023 20:23:28 +0000 Subject: [PATCH 15/16] hash table name for hidden attrs --- datajoint/declare.py | 8 ++++---- tests_old/test_declare.py | 12 ++++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/datajoint/declare.py b/datajoint/declare.py index 0ae2ff9cd..41ed53fc2 100644 --- a/datajoint/declare.py +++ b/datajoint/declare.py @@ -5,6 +5,7 @@ import re import pyparsing as pp import logging +from hashlib import sha1 from .errors import DataJointError, _support_filepath_types, FILEPATH_FEATURE_SWITCH from .attribute_adapter import get_adapter from .condition import translate_attribute @@ -19,7 +20,6 @@ METADATA_ATTRIBUTES_SQL = [ "`_{full_table_name}_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP" ] -LONGEST_METADATA_ATTRIBUTE = len("__timestamp") TYPE_PATTERN = { k: re.compile(v, re.I) @@ -315,9 +315,9 @@ def declare(full_table_name, definition, context): attribute_sql.extend( [ attr.format( - full_table_name=full_table_name.replace("`", "")[ - 0 : 64 - LONGEST_METADATA_ATTRIBUTE - ] + full_table_name=sha1( + full_table_name.replace("`", "").encode("utf-8") + ).hexdigest() ) for attr in METADATA_ATTRIBUTES_SQL ] diff --git a/tests_old/test_declare.py b/tests_old/test_declare.py index 4a223c649..8c2c2caff 100644 --- a/tests_old/test_declare.py +++ b/tests_old/test_declare.py @@ -345,10 +345,14 @@ class WithSuchALongPartNameThatItCrashesMySQL(dj.Part): @staticmethod def test_hidden_attributes(): assert ( - list(Experiment().heading._attributes.keys())[-1] - == "_djtest_test1._experiment_timestamp" + list(Experiment().heading._attributes.keys())[-1].split("_")[2] + == "timestamp" ) assert ( - list(Experiment().heading.attributes.keys())[-1] - != "_djtest_test1._experiment_timestamp" + len([a for a in Experiment().heading._attributes.values() if a.is_hidden]) + != 0 + ) + assert ( + len([a for a in Experiment().heading.attributes.values() if a.is_hidden]) + == 0 ) From 4c4bac5685c783a2152f1863f4bad15472c1301e Mon Sep 17 00:00:00 2001 From: A-Baji Date: Fri, 23 Jun 2023 20:26:06 +0000 Subject: [PATCH 16/16] move to local variable --- datajoint/declare.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/datajoint/declare.py b/datajoint/declare.py index 41ed53fc2..0c9d0266b 100644 --- a/datajoint/declare.py +++ b/datajoint/declare.py @@ -17,9 +17,6 @@ "NULL", } # SQL literals to be used without quotes (case insensitive) EXTERNAL_TABLE_ROOT = "~external" -METADATA_ATTRIBUTES_SQL = [ - "`_{full_table_name}_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP" -] TYPE_PATTERN = { k: re.compile(v, re.I) @@ -312,6 +309,10 @@ def declare(full_table_name, definition, context): index_sql, external_stores, ) = prepare_declare(definition, context) + + metadata_attr_sql = [ + "`_{full_table_name}_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP" + ] attribute_sql.extend( [ attr.format( @@ -319,7 +320,7 @@ def declare(full_table_name, definition, context): full_table_name.replace("`", "").encode("utf-8") ).hexdigest() ) - for attr in METADATA_ATTRIBUTES_SQL + for attr in metadata_attr_sql ] )