diff --git a/pyproject.toml b/pyproject.toml index 29d6ba975..387bbfdfd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -434,7 +434,7 @@ select = [ "PIE", ] #select = ["E", "F", "RUF100", "W", "I001"] -ignore = ["E501", "S101", "SIM118", "SIM108", "PLR2004", "PLR0913", "S110", "PIE810", "PLR0911", "PLR0915", "PLR0912", "D", "D401", "PLW0603", "PLR5501", "PLW2901", "S603"] +ignore = ["E501", "S101", "SIM118", "SIM108", "PLR2004", "PLR0913", "S110", "PIE810", "PLR0911", "PLR0915", "PLR0912", "D", "D401", "PLW0603", "PLR5501", "PLW2901", "S603", "S608"] per-file-ignores = { } diff --git a/src/kiara/defaults.py b/src/kiara/defaults.py index 59ca1d10e..16a7986b6 100644 --- a/src/kiara/defaults.py +++ b/src/kiara/defaults.py @@ -310,3 +310,45 @@ class CHUNK_COMPRESSION_TYPE(Enum): ARCHIVE_NAME_MARKER = "archive_name" DATA_ARCHIVE_DEFAULT_VALUE_MARKER = "default_value" +TABLE_NAME_ARCHIVE_METADATA = "archive_metadata" +TABLE_NAME_DATA_METADATA = "data_value_metadata" +TABLE_NAME_DATA_SERIALIZATION_METADATA = "data_serialization_metadata" +TABLE_NAME_DATA_CHUNKS = "data_chunks" +TABLE_NAME_DATA_PEDIGREE = "data_value_pedigree" +TABLE_NAME_DATA_DESTINIES = "data_value_destiny" +REQUIRED_TABLES_DATA_ARCHIVE = { + TABLE_NAME_ARCHIVE_METADATA, + TABLE_NAME_DATA_METADATA, + TABLE_NAME_DATA_SERIALIZATION_METADATA, + TABLE_NAME_DATA_CHUNKS, + TABLE_NAME_DATA_PEDIGREE, + TABLE_NAME_DATA_DESTINIES, +} + +TABLE_NAME_ALIASES = "aliases" +REQUIRED_TABLES_ALIAS_ARCHIVE = { + TABLE_NAME_ARCHIVE_METADATA, + TABLE_NAME_ALIASES, +} + +TABLE_NAME_JOB_RECORDS = "job_records" +REQUIRED_TABLES_JOB_ARCHIVE = { + TABLE_NAME_ARCHIVE_METADATA, + TABLE_NAME_JOB_RECORDS, +} + +TABLE_NAME_METADATA = "metadata" +TABLE_NAME_METADATA_SCHEMAS = "metadata_schemas" +TABLE_NAME_METADATA_REFERENCES = "metadata_references" +REQUIRED_TABLES_METADATA = { + TABLE_NAME_ARCHIVE_METADATA, + TABLE_NAME_METADATA, + TABLE_NAME_METADATA_SCHEMAS, + TABLE_NAME_METADATA_REFERENCES, +} + + +ALL_REQUIRED_TABLES = set(REQUIRED_TABLES_DATA_ARCHIVE) +ALL_REQUIRED_TABLES.update(REQUIRED_TABLES_ALIAS_ARCHIVE) +ALL_REQUIRED_TABLES.update(REQUIRED_TABLES_JOB_ARCHIVE) +ALL_REQUIRED_TABLES.update(REQUIRED_TABLES_METADATA) diff --git a/src/kiara/registries/aliases/sqlite_store.py b/src/kiara/registries/aliases/sqlite_store.py index b938f3bdd..00953faae 100644 --- a/src/kiara/registries/aliases/sqlite_store.py +++ b/src/kiara/registries/aliases/sqlite_store.py @@ -6,6 +6,11 @@ from sqlalchemy import text from sqlalchemy.engine import Engine +from kiara.defaults import ( + REQUIRED_TABLES_ALIAS_ARCHIVE, + TABLE_NAME_ALIASES, + TABLE_NAME_ARCHIVE_METADATA, +) from kiara.registries import SqliteArchiveConfig from kiara.registries.aliases import AliasArchive, AliasStore from kiara.utils.dates import get_current_time_incl_timezone @@ -36,9 +41,7 @@ def _load_archive_config( tables = {x[0] for x in cursor.fetchall()} con.close() - required_tables = { - "aliases", - } + required_tables = REQUIRED_TABLES_ALIAS_ARCHIVE if not required_tables.issubset(tables): return None @@ -66,7 +69,7 @@ def __init__( def _retrieve_archive_metadata(self) -> Mapping[str, Any]: - sql = text("SELECT key, value FROM archive_metadata") + sql = text(f"SELECT key, value FROM {TABLE_NAME_ARCHIVE_METADATA}") with self.sqlite_engine.connect() as connection: result = connection.execute(sql) @@ -104,8 +107,8 @@ def sqlite_engine(self) -> "Engine": use_wal_mode=self._use_wal_mode, ) - create_table_sql = """ -CREATE TABLE IF NOT EXISTS aliases ( + create_table_sql = f""" +CREATE TABLE IF NOT EXISTS {TABLE_NAME_ALIASES} ( alias TEXT PRIMARY KEY, value_id TEXT NOT NULL, alias_created TEXT NOT NULL @@ -122,7 +125,7 @@ def sqlite_engine(self) -> "Engine": def find_value_id_for_alias(self, alias: str) -> Union[uuid.UUID, None]: - sql = text("SELECT value_id FROM aliases WHERE alias = :alias") + sql = text(f"SELECT value_id FROM {TABLE_NAME_ALIASES} WHERE alias = :alias") with self.sqlite_engine.connect() as connection: result = connection.execute(sql, {"alias": alias}) row = result.fetchone() @@ -132,14 +135,14 @@ def find_value_id_for_alias(self, alias: str) -> Union[uuid.UUID, None]: def find_aliases_for_value_id(self, value_id: uuid.UUID) -> Union[Set[str], None]: - sql = text("SELECT alias FROM aliases WHERE value_id = :value_id") + sql = text(f"SELECT alias FROM {TABLE_NAME_ALIASES} WHERE value_id = :value_id") with self.sqlite_engine.connect() as connection: result = connection.execute(sql, {"value_id": str(value_id)}) return {row[0] for row in result} def retrieve_all_aliases(self) -> Union[Mapping[str, uuid.UUID], None]: - sql = text("SELECT alias, value_id FROM aliases") + sql = text(f"SELECT alias, value_id FROM {TABLE_NAME_ALIASES}") with self.sqlite_engine.connect() as connection: result = connection.execute(sql) return {row[0]: uuid.UUID(row[1]) for row in result} @@ -173,9 +176,7 @@ def _load_archive_config( tables = {x[0] for x in cursor.fetchall()} con.close() - required_tables = { - "aliases", - } + required_tables = REQUIRED_TABLES_ALIAS_ARCHIVE if not required_tables.issubset(tables): return None @@ -187,7 +188,7 @@ def _set_archive_metadata_value(self, key: str, value: Any): """Set custom metadata for the archive.""" sql = text( - "INSERT OR REPLACE INTO archive_metadata (key, value) VALUES (:key, :value)" + f"INSERT OR REPLACE INTO {TABLE_NAME_ARCHIVE_METADATA} (key, value) VALUES (:key, :value)" ) with self.sqlite_engine.connect() as conn: params = {"key": key, "value": value} @@ -199,7 +200,7 @@ def register_aliases(self, value_id: uuid.UUID, *aliases: str): alias_created = get_current_time_incl_timezone().isoformat() sql = text( - "INSERT OR REPLACE INTO aliases (alias, value_id, alias_created) VALUES (:alias, :value_id, :alias_created)" + f"INSERT OR REPLACE INTO {TABLE_NAME_ALIASES} (alias, value_id, alias_created) VALUES (:alias, :value_id, :alias_created)" ) with self.sqlite_engine.connect() as connection: diff --git a/src/kiara/registries/data/data_store/__init__.py b/src/kiara/registries/data/data_store/__init__.py index 172be3975..4a9bbf473 100644 --- a/src/kiara/registries/data/data_store/__init__.py +++ b/src/kiara/registries/data/data_store/__init__.py @@ -198,31 +198,31 @@ def has_value(self, value_id: uuid.UUID) -> bool: return False return value_id in all_value_ids - def retrieve_environment_details( - self, env_type: str, env_hash: str - ) -> Mapping[str, Any]: - """ - Retrieve the environment details with the specified type and hash. - - The environment is stored by the data store as a dictionary, including it's schema, not as the actual Python model. - This is to make sure it can still be loaded later on, in case the Python model has changed in later versions. - """ - cached = self._env_cache.get(env_type, {}).get(env_hash, None) - if cached is not None: - return cached - - env = self._retrieve_environment_details(env_type=env_type, env_hash=env_hash) - self._env_cache.setdefault(env_type, {})[env_hash] = env - return env - - @abc.abstractmethod - def _retrieve_environment_details( - self, env_type: str, env_hash: str - ) -> Mapping[str, Any]: - """Retrieve the environment details with the specified type and hash. - - Each store needs to implement this so environemnt details related to a value can be retrieved later on. Since in most cases the environment details will not change, a lookup is more efficient than having to store the full information with each value. - """ + # def retrieve_environment_details( + # self, env_type: str, env_hash: str + # ) -> Mapping[str, Any]: + # """ + # Retrieve the environment details with the specified type and hash. + # + # The environment is stored by the data store as a dictionary, including it's schema, not as the actual Python model. + # This is to make sure it can still be loaded later on, in case the Python model has changed in later versions. + # """ + # cached = self._env_cache.get(env_type, {}).get(env_hash, None) + # if cached is not None: + # return cached + # + # env = self._retrieve_environment_details(env_type=env_type, env_hash=env_hash) + # self._env_cache.setdefault(env_type, {})[env_hash] = env + # return env + # + # @abc.abstractmethod + # def _retrieve_environment_details( + # self, env_type: str, env_hash: str + # ) -> Mapping[str, Any]: + # """Retrieve the environment details with the specified type and hash. + # + # Each store needs to implement this so environemnt details related to a value can be retrieved later on. Since in most cases the environment details will not change, a lookup is more efficient than having to store the full information with each value. + # """ def find_values(self, matcher: ValueMatcher) -> Iterable[Value]: raise NotImplementedError() diff --git a/src/kiara/registries/data/data_store/filesystem_store.py b/src/kiara/registries/data/data_store/filesystem_store.py index 9acba8a6a..d57267965 100644 --- a/src/kiara/registries/data/data_store/filesystem_store.py +++ b/src/kiara/registries/data/data_store/filesystem_store.py @@ -182,20 +182,20 @@ def get_path( result.mkdir(parents=True, exist_ok=True) return result - def _retrieve_environment_details( - self, env_type: str, env_hash: str - ) -> Mapping[str, Any]: - - base_path = self.get_path(entity_type=EntityType.ENVIRONMENT) - env_details_file = base_path / f"{env_type}_{env_hash}.json" - - if not env_details_file.exists(): - raise Exception( - f"Can't load environment details, file does not exist: {env_details_file.as_posix()}" - ) - - environment: Mapping[str, Any] = orjson.loads(env_details_file.read_text()) - return environment + # def _retrieve_environment_details( + # self, env_type: str, env_hash: str + # ) -> Mapping[str, Any]: + # + # base_path = self.get_path(entity_type=EntityType.ENVIRONMENT) + # env_details_file = base_path / f"{env_type}_{env_hash}.json" + # + # if not env_details_file.exists(): + # raise Exception( + # f"Can't load environment details, file does not exist: {env_details_file.as_posix()}" + # ) + # + # environment: Mapping[str, Any] = orjson.loads(env_details_file.read_text()) + # return environment def retrieve_all_job_hashes( self, diff --git a/src/kiara/registries/data/data_store/sqlite_store.py b/src/kiara/registries/data/data_store/sqlite_store.py index 279bc199a..6c7188ec9 100644 --- a/src/kiara/registries/data/data_store/sqlite_store.py +++ b/src/kiara/registries/data/data_store/sqlite_store.py @@ -26,6 +26,13 @@ CHUNK_CACHE_DIR_DEPTH, CHUNK_CACHE_DIR_WIDTH, CHUNK_COMPRESSION_TYPE, + REQUIRED_TABLES_DATA_ARCHIVE, + TABLE_NAME_ARCHIVE_METADATA, + TABLE_NAME_DATA_CHUNKS, + TABLE_NAME_DATA_DESTINIES, + TABLE_NAME_DATA_METADATA, + TABLE_NAME_DATA_PEDIGREE, + TABLE_NAME_DATA_SERIALIZATION_METADATA, ) from kiara.models.values.value import PersistedData, Value from kiara.registries import ( @@ -69,15 +76,7 @@ def _load_archive_config( tables = {x[0] for x in cursor.fetchall()} con.close() - required_tables = { - "values_pedigree", - "values_destinies", - "archive_metadata", - "persisted_values", - "values_metadata", - "values_data", - "environments", - } + required_tables = REQUIRED_TABLES_DATA_ARCHIVE if not required_tables.issubset(tables): return None @@ -110,7 +109,7 @@ def __init__( def _retrieve_archive_metadata(self) -> Mapping[str, Any]: - sql = text("SELECT key, value FROM archive_metadata") + sql = text(f"SELECT key, value FROM {TABLE_NAME_ARCHIVE_METADATA}") with self.sqlite_engine.connect() as connection: result = connection.execute(sql) @@ -171,8 +170,8 @@ def sqlite_engine(self) -> "Engine": use_wal_mode=self._use_wal_mode, ) - create_table_sql = """ -CREATE TABLE IF NOT EXISTS values_metadata ( + create_table_sql = f""" +CREATE TABLE IF NOT EXISTS {TABLE_NAME_DATA_METADATA} ( value_id TEXT PRIMARY KEY, value_hash TEXT NOT NULL, value_size INTEGER NOT NULL, @@ -180,32 +179,26 @@ def sqlite_engine(self) -> "Engine": data_type_name TEXT NOT NULL, value_metadata TEXT NOT NULL ); -CREATE TABLE IF NOT EXISTS persisted_values ( +CREATE TABLE IF NOT EXISTS {TABLE_NAME_DATA_SERIALIZATION_METADATA} ( value_id TEXT PRIMARY KEY, value_hash TEXT NOT NULL, value_size INTEGER NOT NULL, data_type_name TEXT NOT NULL, persisted_value_metadata TEXT NOT NULL ); -CREATE TABLE IF NOT EXISTS values_data ( +CREATE TABLE IF NOT EXISTS {TABLE_NAME_DATA_CHUNKS} ( chunk_id TEXT PRIMARY KEY, chunk_data BLOB NOT NULL, compression_type INTEGER NULL ); -CREATE TABLE IF NOT EXISTS values_pedigree ( +CREATE TABLE IF NOT EXISTS {TABLE_NAME_DATA_PEDIGREE} ( value_id TEXT NOT NULL PRIMARY KEY, pedigree TEXT NOT NULL ); -CREATE TABLE IF NOT EXISTS values_destinies ( +CREATE TABLE IF NOT EXISTS {TABLE_NAME_DATA_DESTINIES} ( value_id TEXT NOT NULL, destiny_name TEXT NOT NULL ); -CREATE TABLE IF NOT EXISTS environments ( - environment_type TEXT NOT NULL, - environment_hash TEXT NOT NULL, - environment_data TEXT NOT NULL, - PRIMARY KEY (environment_type, environment_hash) -); """ with self._cached_engine.begin() as connection: @@ -221,7 +214,7 @@ def _retrieve_serialized_value(self, value: Value) -> PersistedData: value_id = str(value.value_id) sql = text( - "SELECT persisted_value_metadata FROM persisted_values WHERE value_id = :value_id" + f"SELECT persisted_value_metadata FROM {TABLE_NAME_DATA_SERIALIZATION_METADATA} WHERE value_id = :value_id" ) with self.sqlite_engine.connect() as conn: cursor = conn.execute(sql, {"value_id": value_id}) @@ -232,7 +225,7 @@ def _retrieve_serialized_value(self, value: Value) -> PersistedData: def _retrieve_value_details(self, value_id: uuid.UUID) -> Mapping[str, Any]: sql = text( - "SELECT value_metadata FROM values_metadata WHERE value_id = :value_id" + f"SELECT value_metadata FROM {TABLE_NAME_DATA_METADATA} WHERE value_id = :value_id" ) params = {"value_id": str(value_id)} with self.sqlite_engine.connect() as conn: @@ -241,17 +234,17 @@ def _retrieve_value_details(self, value_id: uuid.UUID) -> Mapping[str, Any]: data: Mapping[str, Any] = orjson.loads(result[0]) return data - def _retrieve_environment_details( - self, env_type: str, env_hash: str - ) -> Mapping[str, Any]: - - sql = text( - "SELECT environment_data FROM environments_data WHERE environment_type = ? AND environment_hash = ?" - ) - with self.sqlite_engine.connect() as conn: - cursor = conn.execute(sql, (env_type, env_hash)) - result = cursor.fetchone() - return result[0] # type: ignore + # def _retrieve_environment_details( + # self, env_type: str, env_hash: str + # ) -> Mapping[str, Any]: + # + # sql = text( + # "SELECT environment_data FROM environments_data WHERE environment_type = ? AND environment_hash = ?" + # ) + # with self.sqlite_engine.connect() as conn: + # cursor = conn.execute(sql, (env_type, env_hash)) + # result = cursor.fetchone() + # return result[0] # type: ignore # def find_values(self, matcher: ValueMatcher) -> Iterable[Value]: # raise NotImplementedError() @@ -274,7 +267,7 @@ def has_value(self, value_id: uuid.UUID) -> bool: """ sql_text = text( - "SELECT EXISTS(SELECT 1 FROM values_metadata WHERE value_id = :value_id)" + f"SELECT EXISTS(SELECT 1 FROM {TABLE_NAME_DATA_METADATA} WHERE value_id = :value_id)" ) with self.sqlite_engine.connect() as conn: result = conn.execute(sql_text, {"value_id": str(value_id)}).scalar() @@ -287,7 +280,7 @@ def _retrieve_all_value_ids( if self._value_id_cache is not None: return self._value_id_cache - sql = text("SELECT value_id FROM values_metadata") + sql = text(f"SELECT value_id FROM {TABLE_NAME_DATA_METADATA}") with self.sqlite_engine.connect() as conn: cursor = conn.execute(sql) result = cursor.fetchall() @@ -297,7 +290,7 @@ def _retrieve_all_value_ids( def retrieve_all_chunk_ids(self) -> Iterable[str]: - sql = text("SELECT chunk_id FROM values_data") + sql = text(f"SELECT chunk_id FROM {TABLE_NAME_DATA_CHUNKS}") with self.sqlite_engine.connect() as conn: cursor = conn.execute(sql) result = cursor.fetchall() @@ -315,7 +308,9 @@ def _find_values_with_hash( if data_type_name is not None: raise NotImplementedError() - sql = text("SELECT value_id FROM values_metadata WHERE value_hash = ?") + sql = text( + f"SELECT value_id FROM {TABLE_NAME_DATA_METADATA} WHERE value_hash = ?" + ) with self.sqlite_engine.connect() as conn: cursor = conn.execute(sql, (value_hash,)) result = cursor.fetchall() @@ -326,7 +321,7 @@ def _find_destinies_for_value( ) -> Union[Mapping[str, uuid.UUID], None]: sql = text( - "SELECT destiny_name FROM values_destinies WHERE value_id = :value_id" + f"SELECT destiny_name FROM {TABLE_NAME_DATA_DESTINIES} WHERE value_id = :value_id" ) params = {"value_id": str(value_id)} with self.sqlite_engine.connect() as conn: @@ -410,14 +405,14 @@ def retrieve_missing_chunks( id_list_str = ", ".join("'" + item + "'" for item in missing_ids) sql = text( - f"""SELECT chunk_id, chunk_data, compression_type FROM values_data + f"""SELECT chunk_id, chunk_data, compression_type FROM {TABLE_NAME_DATA_CHUNKS} WHERE chunk_id in ({id_list_str}) ORDER BY CASE chunk_id {"".join([f"WHEN '{id}' THEN {i} " for i, id in enumerate(missing_ids)])} END - """ # noqa + """ ) result = conn.execute(sql) @@ -537,15 +532,7 @@ def _load_archive_config( tables = {x[0] for x in cursor.fetchall()} con.close() - required_tables = { - "values_pedigree", - "values_destinies", - "archive_metadata", - "persisted_values", - "values_metadata", - "values_data", - "environments", - } + required_tables = REQUIRED_TABLES_DATA_ARCHIVE if not required_tables.issubset(tables): return None @@ -557,7 +544,7 @@ def _set_archive_metadata_value(self, key: str, value: Any): """Set custom metadata for the archive.""" sql = text( - "INSERT OR REPLACE INTO archive_metadata (key, value) VALUES (:key, :value)" + f"INSERT OR REPLACE INTO {TABLE_NAME_ARCHIVE_METADATA} (key, value) VALUES (:key, :value)" ) with self.sqlite_engine.connect() as conn: params = {"key": key, "value": value} @@ -660,7 +647,7 @@ def _persist_chunk( else None ) sql = text( - "INSERT INTO values_data (chunk_id, chunk_data, compression_type) VALUES (:chunk_id, :chunk_data, :compression_type)" + f"INSERT INTO {TABLE_NAME_DATA_CHUNKS} (chunk_id, chunk_data, compression_type) VALUES (:chunk_id, :chunk_data, :compression_type)" ) params = { "chunk_id": chunk_id, @@ -683,7 +670,7 @@ def _persist_stored_value_info(self, value: Value, persisted_value: PersistedDat metadata = persisted_value.model_dump_json() sql = text( - "INSERT INTO persisted_values (value_id, value_hash, value_size, data_type_name, persisted_value_metadata) VALUES (:value_id, :value_hash, :value_size, :data_type_name, :metadata)" + f"INSERT INTO {TABLE_NAME_DATA_SERIALIZATION_METADATA} (value_id, value_hash, value_size, data_type_name, persisted_value_metadata) VALUES (:value_id, :value_hash, :value_size, :data_type_name, :metadata)" ) with self.sqlite_engine.connect() as conn: @@ -709,7 +696,7 @@ def _persist_value_details(self, value: Value): metadata = value.model_dump_json() sql = text( - "INSERT INTO values_metadata (value_id, value_hash, value_size, value_created, data_type_name, value_metadata) VALUES (:value_id, :value_hash, :value_size, :value_created, :data_type_name, :metadata)" + f"INSERT INTO {TABLE_NAME_DATA_METADATA} (value_id, value_hash, value_size, value_created, data_type_name, value_metadata) VALUES (:value_id, :value_hash, :value_size, :value_created, :data_type_name, :metadata)" ) with self.sqlite_engine.connect() as conn: params = { @@ -732,7 +719,7 @@ def _persist_destiny_backlinks(self, value: Value): for destiny_value_id, destiny_name in value.destiny_backlinks.items(): sql = text( - "INSERT INTO values_destinies (value_id, destiny_name) VALUES (:value_id, :destiny_name)" + f"INSERT INTO {TABLE_NAME_DATA_DESTINIES} (value_id, destiny_name) VALUES (:value_id, :destiny_name)" ) params = { "value_id": value_id, @@ -748,7 +735,7 @@ def _persist_value_pedigree(self, value: Value): pedigree = value.pedigree.manifest_data_as_json() sql = text( - "INSERT INTO values_pedigree (value_id, pedigree) VALUES (:value_id, :pedigree)" + f"INSERT INTO {TABLE_NAME_DATA_PEDIGREE} (value_id, pedigree) VALUES (:value_id, :pedigree)" ) with self.sqlite_engine.connect() as conn: params = {"value_id": value_id, "pedigree": pedigree} diff --git a/src/kiara/registries/jobs/job_store/sqlite_store.py b/src/kiara/registries/jobs/job_store/sqlite_store.py index e8aac33b8..3af076f3e 100644 --- a/src/kiara/registries/jobs/job_store/sqlite_store.py +++ b/src/kiara/registries/jobs/job_store/sqlite_store.py @@ -8,6 +8,11 @@ from sqlalchemy import text from sqlalchemy.engine import Engine +from kiara.defaults import ( + REQUIRED_TABLES_JOB_ARCHIVE, + TABLE_NAME_ARCHIVE_METADATA, + TABLE_NAME_JOB_RECORDS, +) from kiara.models.module.jobs import JobMatcher, JobRecord from kiara.registries import ArchiveDetails, SqliteArchiveConfig from kiara.registries.jobs import JobArchive, JobStore @@ -39,9 +44,7 @@ def _load_archive_config( tables = {x[0] for x in cursor.fetchall()} con.close() - required_tables = { - "job_records", - } + required_tables = REQUIRED_TABLES_JOB_ARCHIVE if not required_tables.issubset(tables): return None @@ -78,7 +81,7 @@ def __init__( def _retrieve_archive_metadata(self) -> Mapping[str, Any]: - sql = text("SELECT key, value FROM archive_metadata") + sql = text(f"SELECT key, value FROM {TABLE_NAME_ARCHIVE_METADATA}") with self.sqlite_engine.connect() as connection: result = connection.execute(sql) @@ -116,8 +119,8 @@ def sqlite_engine(self) -> "Engine": use_wal_mode=self._use_wal_mode, ) - create_table_sql = """ -CREATE TABLE IF NOT EXISTS job_records ( + create_table_sql = f""" +CREATE TABLE IF NOT EXISTS {TABLE_NAME_JOB_RECORDS} ( job_id TEXT PRIMARY KEY, job_hash TEXT TEXT NOT NULL, job_submitted TEXT NOT NULL, @@ -139,7 +142,9 @@ def sqlite_engine(self) -> "Engine": def _retrieve_record_for_job_hash(self, job_hash: str) -> Union[JobRecord, None]: - sql = text("SELECT job_metadata FROM job_records WHERE job_hash = :job_hash") + sql = text( + f"SELECT job_metadata FROM {TABLE_NAME_JOB_RECORDS} WHERE job_hash = :job_hash" + ) params = {"job_hash": job_hash} with self.sqlite_engine.connect() as connection: @@ -159,7 +164,7 @@ def _retrieve_all_job_ids(self) -> Mapping[uuid.UUID, datetime]: """ sql = text( - "SELECT job_id, job_submitted FROM job_records ORDER BY job_submitted DESC;" + f"SELECT job_id, job_submitted FROM {TABLE_NAME_JOB_RECORDS} ORDER BY job_submitted DESC;" ) with self.sqlite_engine.connect() as connection: @@ -168,7 +173,9 @@ def _retrieve_all_job_ids(self) -> Mapping[uuid.UUID, datetime]: def _retrieve_record_for_job_id(self, job_id: uuid.UUID) -> Union[JobRecord, None]: - sql = text("SELECT job_metadata FROM job_records WHERE job_id = :job_id") + sql = text( + f"SELECT job_metadata FROM {TABLE_NAME_JOB_RECORDS} WHERE job_id = :job_id" + ) params = {"job_id": str(job_id)} @@ -217,7 +224,7 @@ def _retrieve_matching_job_records( "Job matcher 'produced_outputs' not implemented yet" ) - sql_query = "SELECT job_id, job_metadata FROM job_records" + sql_query = f"SELECT job_id, job_metadata FROM {TABLE_NAME_JOB_RECORDS}" if query_conditions: sql_query += " WHERE " @@ -247,22 +254,22 @@ def retrieve_all_job_hashes( if not manifest_hash: if not inputs_id_hash: - sql = text("SELECT job_hash FROM job_records") + sql = text(f"SELECT job_hash FROM {TABLE_NAME_JOB_RECORDS}") params = {} else: sql = text( - "SELECT job_hash FROM job_records WHERE inputs_hash = :inputs_hash" + f"SELECT job_hash FROM {TABLE_NAME_JOB_RECORDS} WHERE inputs_hash = :inputs_hash" ) params = {"inputs_hash": inputs_id_hash} else: if not inputs_id_hash: sql = text( - "SELECT job_hash FROM job_records WHERE manifest_hash = :manifest_hash" + f"SELECT job_hash FROM {TABLE_NAME_JOB_RECORDS} WHERE manifest_hash = :manifest_hash" ) params = {"manifest_hash": manifest_hash} else: sql = text( - "SELECT job_hash FROM job_records WHERE manifest_hash = :manifest_hash AND inputs_hash = :inputs_hash" + f"SELECT job_hash FROM {TABLE_NAME_JOB_RECORDS} WHERE manifest_hash = :manifest_hash AND inputs_hash = :inputs_hash" ) params = {"manifest_hash": manifest_hash, "inputs_hash": inputs_id_hash} @@ -276,7 +283,7 @@ def _delete_archive(self): def get_archive_details(self) -> ArchiveDetails: - all_job_records_sql = text("SELECT COUNT(*) FROM job_records") + all_job_records_sql = text(f"SELECT COUNT(*) FROM {TABLE_NAME_JOB_RECORDS}") with self.sqlite_engine.connect() as connection: result = connection.execute(all_job_records_sql) @@ -310,9 +317,7 @@ def _load_archive_config( tables = {x[0] for x in cursor.fetchall()} con.close() - required_tables = { - "job_records", - } + required_tables = REQUIRED_TABLES_JOB_ARCHIVE if not required_tables.issubset(tables): return None @@ -332,7 +337,7 @@ def store_job_record(self, job_record: JobRecord): job_submitted = job_record.job_submitted.isoformat() sql = text( - "INSERT OR IGNORE INTO job_records(job_id, job_submitted, job_hash, manifest_hash, input_ids_hash, inputs_data_hash, job_metadata) VALUES (:job_id, :job_submitted, :job_hash, :manifest_hash, :input_ids_hash, :inputs_data_hash, :job_metadata)" + f"INSERT OR IGNORE INTO {TABLE_NAME_JOB_RECORDS}(job_id, job_submitted, job_hash, manifest_hash, input_ids_hash, inputs_data_hash, job_metadata) VALUES (:job_id, :job_submitted, :job_hash, :manifest_hash, :input_ids_hash, :inputs_data_hash, :job_metadata)" ) params = { "job_id": str(job_record.job_id), @@ -353,7 +358,7 @@ def _set_archive_metadata_value(self, key: str, value: Any): """Set custom metadata for the archive.""" sql = text( - "INSERT OR REPLACE INTO archive_metadata (key, value) VALUES (:key, :value)" + f"INSERT OR REPLACE INTO {TABLE_NAME_ARCHIVE_METADATA} (key, value) VALUES (:key, :value)" ) with self.sqlite_engine.connect() as conn: params = {"key": key, "value": value} diff --git a/src/kiara/registries/metadata/metadata_store/sqlite_store.py b/src/kiara/registries/metadata/metadata_store/sqlite_store.py index 39358c1dc..1cdb63af3 100644 --- a/src/kiara/registries/metadata/metadata_store/sqlite_store.py +++ b/src/kiara/registries/metadata/metadata_store/sqlite_store.py @@ -15,16 +15,19 @@ from sqlalchemy import text from sqlalchemy.engine import Engine +from kiara.defaults import ( + REQUIRED_TABLES_METADATA, + TABLE_NAME_ARCHIVE_METADATA, + TABLE_NAME_METADATA, + TABLE_NAME_METADATA_REFERENCES, + TABLE_NAME_METADATA_SCHEMAS, +) from kiara.exceptions import KiaraException from kiara.registries import ArchiveDetails, SqliteArchiveConfig from kiara.registries.metadata import MetadataArchive, MetadataMatcher, MetadataStore from kiara.utils.dates import get_current_time_incl_timezone from kiara.utils.db import create_archive_engine, delete_archive_db -REQUIRED_METADATA_TABLES = { - "metadata", -} - class SqliteMetadataArchive(MetadataArchive): @@ -51,7 +54,7 @@ def _load_archive_config( tables = {x[0] for x in cursor.fetchall()} con.close() - if not REQUIRED_METADATA_TABLES.issubset(tables): + if not REQUIRED_TABLES_METADATA.issubset(tables): return None # config = SqliteArchiveConfig(sqlite_db_path=store_uri) @@ -87,7 +90,7 @@ def __init__( def _retrieve_archive_metadata(self) -> Mapping[str, Any]: - sql = text("SELECT key, value FROM archive_metadata") + sql = text(f"SELECT key, value FROM {TABLE_NAME_ARCHIVE_METADATA}") with self.sqlite_engine.connect() as connection: result = connection.execute(sql) @@ -125,13 +128,13 @@ def sqlite_engine(self) -> "Engine": use_wal_mode=self._use_wal_mode, ) - create_table_sql = """ -CREATE TABLE IF NOT EXISTS metadata_schemas ( + create_table_sql = f""" +CREATE TABLE IF NOT EXISTS {TABLE_NAME_METADATA_SCHEMAS} ( model_schema_hash TEXT PRIMARY KEY, model_type_id TEXT NOT NULL, model_schema TEXT NOT NULL ); -CREATE TABLE IF NOT EXISTS metadata ( +CREATE TABLE IF NOT EXISTS {TABLE_NAME_METADATA} ( metadata_item_id TEXT PRIMARY KEY, metadata_item_created TEXT NOT NULL, metadata_item_key TEXT NOT NULL, @@ -142,7 +145,7 @@ def sqlite_engine(self) -> "Engine": FOREIGN KEY (model_schema_hash) REFERENCES metadata_schemas (model_schema_hash), UNIQUE (metadata_item_key, metadata_item_hash) ); -CREATE TABLE IF NOT EXISTS metadata_references ( +CREATE TABLE IF NOT EXISTS {TABLE_NAME_METADATA_REFERENCES} ( reference_item_type TEXT NOT NULL, reference_item_key TEXT NOT NULL, reference_item_id TEXT NOT NULL, @@ -168,17 +171,17 @@ def _retrieve_metadata_item_with_hash( if not key: sql = text( - """ + f""" SELECT m.model_type_id, m.metadata_value - FROM metadata m + FROM {TABLE_NAME_METADATA} m WHERE m.metadata_item_hash = :item_hash """ ) else: sql = text( - """ + f""" SELECT m.model_type_id, m.metadata_value - FROM metadata m + FROM {TABLE_NAME_METADATA} m WHERE m.metadata_item_hash = :item_hash AND m.metadata_item_key = :key """ ) @@ -224,7 +227,7 @@ def _find_matching_metadata_and_ref_items( metadata_fields_str += ", :result_type as result_type" - sql_string = f"SELECT {metadata_fields_str} FROM metadata m " # noqa + sql_string = f"SELECT {metadata_fields_str} FROM {TABLE_NAME_METADATA} m " conditions = [] params = {"result_type": "metadata_item"} @@ -289,7 +292,7 @@ def _find_matching_metadata_and_ref_items( (f"r.{x}" for x in reference_item_result_fields) ) - ref_sql_string = f"SELECT {reference_fields_str}, :result_type as result_type FROM metadata_references r" # noqa + ref_sql_string = f"SELECT {reference_fields_str}, :result_type as result_type FROM metadata_references r" ref_params = {"result_type": "metadata_ref_item"} ref_conditions = [] @@ -345,10 +348,10 @@ def _retrieve_referenced_metadata_item_data( ) -> Union[Tuple[str, Mapping[str, Any]], None]: sql = text( - """ + f""" SELECT m.model_type_id, m.metadata_value - FROM metadata m - JOIN metadata_references r ON m.metadata_item_id = r.metadata_item_id + FROM {TABLE_NAME_METADATA} m + JOIN {TABLE_NAME_METADATA_REFERENCES} r ON m.metadata_item_id = r.metadata_item_id WHERE r.reference_item_type = :reference_type AND r.reference_item_key = :reference_key AND r.reference_item_id = :reference_id and m.metadata_item_key = :key """ ) @@ -385,8 +388,10 @@ def _delete_archive(self): def get_archive_details(self) -> ArchiveDetails: - all_metadata_items_sql = text("SELECT COUNT(*) FROM metadata") - all_references_sql = text("SELECT COUNT(*) FROM metadata_references") + all_metadata_items_sql = text(f"SELECT COUNT(*) FROM {TABLE_NAME_METADATA}") + all_references_sql = text( + f"SELECT COUNT(*) FROM {TABLE_NAME_METADATA_REFERENCES}" + ) with self.sqlite_engine.connect() as connection: result = connection.execute(all_metadata_items_sql) @@ -427,7 +432,7 @@ def _load_archive_config( tables = {x[0] for x in cursor.fetchall()} con.close() - if not REQUIRED_METADATA_TABLES.issubset(tables): + if not REQUIRED_TABLES_METADATA.issubset(tables): return None # config = SqliteArchiveConfig(sqlite_db_path=store_uri) @@ -437,7 +442,7 @@ def _set_archive_metadata_value(self, key: str, value: Any): """Set custom metadata for the archive.""" sql = text( - "INSERT OR REPLACE INTO archive_metadata (key, value) VALUES (:key, :value)" + f"INSERT OR REPLACE INTO {TABLE_NAME_ARCHIVE_METADATA} (key, value) VALUES (:key, :value)" ) with self.sqlite_engine.connect() as conn: params = {"key": key, "value": value} @@ -449,7 +454,7 @@ def _store_metadata_schema( ): sql = text( - "INSERT OR IGNORE INTO metadata_schemas (model_schema_hash, model_type_id, model_schema) VALUES (:model_schema_hash, :model_type_id, :model_schema)" + f"INSERT OR IGNORE INTO {TABLE_NAME_METADATA_SCHEMAS} (model_schema_hash, model_type_id, model_schema) VALUES (:model_schema_hash, :model_type_id, :model_schema)" ) params = { "model_schema_hash": model_schema_hash, @@ -474,7 +479,7 @@ def _store_metadata_item( metadata_item_created = get_current_time_incl_timezone().isoformat() sql = text( - "INSERT OR IGNORE INTO metadata (metadata_item_id, metadata_item_created, metadata_item_key, metadata_item_hash, model_type_id, model_schema_hash, metadata_value) VALUES (:metadata_item_id, :metadata_item_created, :metadata_item_key, :metadata_item_hash, :model_type_id, :model_schema_hash, :metadata_value)" + f"INSERT OR IGNORE INTO {TABLE_NAME_METADATA} (metadata_item_id, metadata_item_created, metadata_item_key, metadata_item_hash, model_type_id, model_schema_hash, metadata_value) VALUES (:metadata_item_id, :metadata_item_created, :metadata_item_key, :metadata_item_hash, :model_type_id, :model_schema_hash, :metadata_value)" ) metadata_item_id = ID_REGISTRY.generate( @@ -492,7 +497,7 @@ def _store_metadata_item( } query_metadata_id = text( - "SELECT metadata_item_id FROM metadata WHERE metadata_item_key = :metadata_item_key AND metadata_item_hash = :metadata_item_hash" + f"SELECT metadata_item_id FROM {TABLE_NAME_METADATA} WHERE metadata_item_key = :metadata_item_key AND metadata_item_hash = :metadata_item_hash" ) query_metadata_params = { "metadata_item_key": key, @@ -525,7 +530,7 @@ def _store_metadata_reference( else: sql_replace = text( - "DELETE FROM metadata_references WHERE reference_item_type = :reference_item_type AND reference_item_key = :reference_item_key AND reference_item_id = :reference_item_id" + f"DELETE FROM {TABLE_NAME_METADATA_REFERENCES} WHERE reference_item_type = :reference_item_type AND reference_item_key = :reference_item_key AND reference_item_id = :reference_item_id" ) sql_replace_params = { "reference_item_type": reference_item_type, @@ -535,7 +540,7 @@ def _store_metadata_reference( metadata_reference_created = get_current_time_incl_timezone().isoformat() sql_insert = text( - "INSERT INTO metadata_references (reference_item_type, reference_item_key, reference_item_id, reference_created, metadata_item_id) VALUES (:reference_item_type, :reference_item_key, :reference_item_id, :reference_created, :metadata_item_id)" + f"INSERT INTO {TABLE_NAME_METADATA_REFERENCES} (reference_item_type, reference_item_key, reference_item_id, reference_created, metadata_item_id) VALUES (:reference_item_type, :reference_item_key, :reference_item_id, :reference_created, :metadata_item_id)" ) sql_insert_params = { "reference_item_type": reference_item_type, @@ -554,11 +559,11 @@ def _store_metadata_and_ref_items( ): insert_metadata_sql = text( - "INSERT OR IGNORE INTO metadata (metadata_item_id, metadata_item_created, metadata_item_key, metadata_item_hash, model_type_id, model_schema_hash, metadata_value) VALUES (:metadata_item_id, :metadata_item_created, :metadata_item_key, :metadata_item_hash, :model_type_id, :model_schema_hash, :metadata_value)" + f"INSERT OR IGNORE INTO {TABLE_NAME_METADATA} (metadata_item_id, metadata_item_created, metadata_item_key, metadata_item_hash, model_type_id, model_schema_hash, metadata_value) VALUES (:metadata_item_id, :metadata_item_created, :metadata_item_key, :metadata_item_hash, :model_type_id, :model_schema_hash, :metadata_value)" ) insert_ref_sql = text( - "INSERT OR IGNORE INTO metadata_references (reference_item_type, reference_item_key, reference_item_id, reference_created, metadata_item_id) VALUES (:reference_item_type, :reference_item_key, :reference_item_id, :reference_created, :metadata_item_id)" + f"INSERT OR IGNORE INTO {TABLE_NAME_METADATA_REFERENCES} (reference_item_type, reference_item_key, reference_item_id, reference_created, metadata_item_id) VALUES (:reference_item_type, :reference_item_key, :reference_item_id, :reference_created, :metadata_item_id)" ) batch_size = 100 diff --git a/tests/resources/archives/nand_true.0.10.kiarchive b/tests/resources/archives/nand_true.0.10.kiarchive index b2c1e8b03..f598f7da4 100644 Binary files a/tests/resources/archives/nand_true.0.10.kiarchive and b/tests/resources/archives/nand_true.0.10.kiarchive differ diff --git a/tests/resources/archives/nand_true.0.10.kiarchive.json b/tests/resources/archives/nand_true.0.10.kiarchive.json index 7d7b8af09..bf6d238f6 100644 --- a/tests/resources/archives/nand_true.0.10.kiarchive.json +++ b/tests/resources/archives/nand_true.0.10.kiarchive.json @@ -1 +1 @@ -{"type_name":"nand_true.0.10.kiarchive","documentation":{"description":"-- n/a --","doc":null},"authors":{"authors":[]},"context":{"references":{},"tags":[],"labels":{}},"data_archive_info":{"type_name":"2dd3e60a-da65-4c3e-ae41-a66f4aab0cba","documentation":{"description":"-- n/a --","doc":null},"authors":{"authors":[]},"context":{"references":{},"tags":[],"labels":{}},"archive_id":"2dd3e60a-da65-4c3e-ae41-a66f4aab0cba","archive_alias":"nand_true.0.10","archive_type_info":{"type_name":"sqlite_data_archive","documentation":{"description":"-- n/a --","doc":null},"authors":{"authors":[{"name":"Markus Binsteiner","email":"markus@frkl.io"}]},"context":{"references":{"source_repo":{"url":"https://github.com/DHARPA-Project/kiara","desc":"The kiara project git repository."},"documentation":{"url":"https://dharpa.org/kiara_documentation/","desc":"The url for kiara documentation."}},"tags":[],"labels":{"package":"kiara"}},"python_class":{"python_class_name":"SqliteDataArchive","python_module_name":"kiara.registries.data.data_store.sqlite_store","full_name":"kiara.registries.data.data_store.sqlite_store.SqliteDataArchive"},"is_writable":false,"supported_item_types":["data"]},"config":{"sqlite_db_path":"/home/markus/projects/kiara/kiara/tests/resources/archives/nand_true.0.10.kiarchive","use_wal_mode":false},"details":{"no_values":6,"value_ids":["02f09f4f-d79d-4a02-8e33-6eafc41804b1","8009efad-0aa0-4bac-8c53-083914778a94","b0e7db2d-7d94-43c5-a93d-ad28f912e67f","b10f8c8c-bd08-40e1-84b2-d1e9091f5289","c9bba191-bd10-4d1f-9752-b4e3da24f12f","edbd6711-0432-430f-a147-08a6ae9df220"],"dynamic_archive":false,"size":135168},"metadata":{"archive_id":"2dd3e60a-da65-4c3e-ae41-a66f4aab0cba","archive_name":"nand_true.0.10"}},"alias_archive_info":{"type_name":"2dd3e60a-da65-4c3e-ae41-a66f4aab0cba","documentation":{"description":"-- n/a --","doc":null},"authors":{"authors":[]},"context":{"references":{},"tags":[],"labels":{}},"archive_id":"2dd3e60a-da65-4c3e-ae41-a66f4aab0cba","archive_alias":"nand_true.0.10","archive_type_info":{"type_name":"sqlite_alias_archive","documentation":{"description":"-- n/a --","doc":null},"authors":{"authors":[{"name":"Markus Binsteiner","email":"markus@frkl.io"}]},"context":{"references":{"source_repo":{"url":"https://github.com/DHARPA-Project/kiara","desc":"The kiara project git repository."},"documentation":{"url":"https://dharpa.org/kiara_documentation/","desc":"The url for kiara documentation."}},"tags":[],"labels":{"package":"kiara"}},"python_class":{"python_class_name":"SqliteAliasArchive","python_module_name":"kiara.registries.aliases.sqlite_store","full_name":"kiara.registries.aliases.sqlite_store.SqliteAliasArchive"},"is_writable":false,"supported_item_types":["alias"]},"config":{"sqlite_db_path":"/home/markus/projects/kiara/kiara/tests/resources/archives/nand_true.0.10.kiarchive","use_wal_mode":false},"details":{"no_aliases":1,"aliases":["y"],"dynamic_archive":false},"metadata":{"archive_id":"2dd3e60a-da65-4c3e-ae41-a66f4aab0cba","archive_name":"nand_true.0.10"}},"metadata_archive_info":{"type_name":"2dd3e60a-da65-4c3e-ae41-a66f4aab0cba","documentation":{"description":"-- n/a --","doc":null},"authors":{"authors":[]},"context":{"references":{},"tags":[],"labels":{}},"archive_id":"2dd3e60a-da65-4c3e-ae41-a66f4aab0cba","archive_alias":"nand_true.0.10","archive_type_info":{"type_name":"sqlite_metadata_archive","documentation":{"description":"-- n/a --","doc":null},"authors":{"authors":[{"name":"Markus Binsteiner","email":"markus@frkl.io"}]},"context":{"references":{"source_repo":{"url":"https://github.com/DHARPA-Project/kiara","desc":"The kiara project git repository."},"documentation":{"url":"https://dharpa.org/kiara_documentation/","desc":"The url for kiara documentation."}},"tags":[],"labels":{"package":"kiara"}},"python_class":{"python_class_name":"SqliteMetadataArchive","python_module_name":"kiara.registries.metadata.metadata_store.sqlite_store","full_name":"kiara.registries.metadata.metadata_store.sqlite_store.SqliteMetadataArchive"},"is_writable":false,"supported_item_types":["metadata"]},"config":{"sqlite_db_path":"/home/markus/projects/kiara/kiara/tests/resources/archives/nand_true.0.10.kiarchive","use_wal_mode":false},"details":{"no_metadata_items":4,"no_references":1,"dynamic_archive":false},"metadata":{"archive_id":"2dd3e60a-da65-4c3e-ae41-a66f4aab0cba","archive_name":"nand_true.0.10"}},"job_archive_info":{"type_name":"2dd3e60a-da65-4c3e-ae41-a66f4aab0cba","documentation":{"description":"-- n/a --","doc":null},"authors":{"authors":[]},"context":{"references":{},"tags":[],"labels":{}},"archive_id":"2dd3e60a-da65-4c3e-ae41-a66f4aab0cba","archive_alias":"nand_true.0.10","archive_type_info":{"type_name":"sqlite_job_archive","documentation":{"description":"-- n/a --","doc":null},"authors":{"authors":[{"name":"Markus Binsteiner","email":"markus@frkl.io"}]},"context":{"references":{"source_repo":{"url":"https://github.com/DHARPA-Project/kiara","desc":"The kiara project git repository."},"documentation":{"url":"https://dharpa.org/kiara_documentation/","desc":"The url for kiara documentation."}},"tags":[],"labels":{"package":"kiara"}},"python_class":{"python_class_name":"SqliteJobArchive","python_module_name":"kiara.registries.jobs.job_store.sqlite_store","full_name":"kiara.registries.jobs.job_store.sqlite_store.SqliteJobArchive"},"is_writable":false,"supported_item_types":["job_record"]},"config":{"sqlite_db_path":"/home/markus/projects/kiara/kiara/tests/resources/archives/nand_true.0.10.kiarchive","use_wal_mode":false},"details":{"no_job_records":4,"dynamic_archive":false},"metadata":{"archive_id":"2dd3e60a-da65-4c3e-ae41-a66f4aab0cba","archive_name":"nand_true.0.10"}}} \ No newline at end of file +{"type_name":"nand_true.0.10.kiarchive","documentation":{"description":"-- n/a --","doc":null},"authors":{"authors":[]},"context":{"references":{},"tags":[],"labels":{}},"data_archive_info":{"type_name":"1df47c50-058f-4f7a-8cdc-4ccab0ab1da9","documentation":{"description":"-- n/a --","doc":null},"authors":{"authors":[]},"context":{"references":{},"tags":[],"labels":{}},"archive_id":"1df47c50-058f-4f7a-8cdc-4ccab0ab1da9","archive_alias":"nand_true.0.10","archive_type_info":{"type_name":"sqlite_data_archive","documentation":{"description":"-- n/a --","doc":null},"authors":{"authors":[{"name":"Markus Binsteiner","email":"markus@frkl.io"}]},"context":{"references":{"source_repo":{"url":"https://github.com/DHARPA-Project/kiara","desc":"The kiara project git repository."},"documentation":{"url":"https://dharpa.org/kiara_documentation/","desc":"The url for kiara documentation."}},"tags":[],"labels":{"package":"kiara"}},"python_class":{"python_class_name":"SqliteDataArchive","python_module_name":"kiara.registries.data.data_store.sqlite_store","full_name":"kiara.registries.data.data_store.sqlite_store.SqliteDataArchive"},"is_writable":false,"supported_item_types":["data"]},"config":{"sqlite_db_path":"/home/markus/projects/kiara/kiara/tests/resources/archives/nand_true.0.10.kiarchive","use_wal_mode":false},"details":{"no_values":6,"value_ids":["2ed7942b-3719-4102-b42f-134a42969945","468dd2b4-0e0c-46f5-9def-05aa57396f46","4c929a5b-c91a-449e-9d5f-f21124298ea7","5668b4ae-a867-4238-bb34-be79115ff36e","60d0a0b3-1030-40c4-8c51-247fea623d83","7b7cd80a-8acd-4543-aa18-7e27ee124a70"],"dynamic_archive":false,"size":126976},"metadata":{"archive_id":"1df47c50-058f-4f7a-8cdc-4ccab0ab1da9","archive_name":"nand_true.0.10"}},"alias_archive_info":{"type_name":"1df47c50-058f-4f7a-8cdc-4ccab0ab1da9","documentation":{"description":"-- n/a --","doc":null},"authors":{"authors":[]},"context":{"references":{},"tags":[],"labels":{}},"archive_id":"1df47c50-058f-4f7a-8cdc-4ccab0ab1da9","archive_alias":"nand_true.0.10","archive_type_info":{"type_name":"sqlite_alias_archive","documentation":{"description":"-- n/a --","doc":null},"authors":{"authors":[{"name":"Markus Binsteiner","email":"markus@frkl.io"}]},"context":{"references":{"source_repo":{"url":"https://github.com/DHARPA-Project/kiara","desc":"The kiara project git repository."},"documentation":{"url":"https://dharpa.org/kiara_documentation/","desc":"The url for kiara documentation."}},"tags":[],"labels":{"package":"kiara"}},"python_class":{"python_class_name":"SqliteAliasArchive","python_module_name":"kiara.registries.aliases.sqlite_store","full_name":"kiara.registries.aliases.sqlite_store.SqliteAliasArchive"},"is_writable":false,"supported_item_types":["alias"]},"config":{"sqlite_db_path":"/home/markus/projects/kiara/kiara/tests/resources/archives/nand_true.0.10.kiarchive","use_wal_mode":false},"details":{"no_aliases":1,"aliases":["y"],"dynamic_archive":false},"metadata":{"archive_id":"1df47c50-058f-4f7a-8cdc-4ccab0ab1da9","archive_name":"nand_true.0.10"}},"metadata_archive_info":{"type_name":"1df47c50-058f-4f7a-8cdc-4ccab0ab1da9","documentation":{"description":"-- n/a --","doc":null},"authors":{"authors":[]},"context":{"references":{},"tags":[],"labels":{}},"archive_id":"1df47c50-058f-4f7a-8cdc-4ccab0ab1da9","archive_alias":"nand_true.0.10","archive_type_info":{"type_name":"sqlite_metadata_archive","documentation":{"description":"-- n/a --","doc":null},"authors":{"authors":[{"name":"Markus Binsteiner","email":"markus@frkl.io"}]},"context":{"references":{"source_repo":{"url":"https://github.com/DHARPA-Project/kiara","desc":"The kiara project git repository."},"documentation":{"url":"https://dharpa.org/kiara_documentation/","desc":"The url for kiara documentation."}},"tags":[],"labels":{"package":"kiara"}},"python_class":{"python_class_name":"SqliteMetadataArchive","python_module_name":"kiara.registries.metadata.metadata_store.sqlite_store","full_name":"kiara.registries.metadata.metadata_store.sqlite_store.SqliteMetadataArchive"},"is_writable":false,"supported_item_types":["metadata"]},"config":{"sqlite_db_path":"/home/markus/projects/kiara/kiara/tests/resources/archives/nand_true.0.10.kiarchive","use_wal_mode":false},"details":{"no_metadata_items":4,"no_references":1,"dynamic_archive":false},"metadata":{"archive_id":"1df47c50-058f-4f7a-8cdc-4ccab0ab1da9","archive_name":"nand_true.0.10"}},"job_archive_info":{"type_name":"1df47c50-058f-4f7a-8cdc-4ccab0ab1da9","documentation":{"description":"-- n/a --","doc":null},"authors":{"authors":[]},"context":{"references":{},"tags":[],"labels":{}},"archive_id":"1df47c50-058f-4f7a-8cdc-4ccab0ab1da9","archive_alias":"nand_true.0.10","archive_type_info":{"type_name":"sqlite_job_archive","documentation":{"description":"-- n/a --","doc":null},"authors":{"authors":[{"name":"Markus Binsteiner","email":"markus@frkl.io"}]},"context":{"references":{"source_repo":{"url":"https://github.com/DHARPA-Project/kiara","desc":"The kiara project git repository."},"documentation":{"url":"https://dharpa.org/kiara_documentation/","desc":"The url for kiara documentation."}},"tags":[],"labels":{"package":"kiara"}},"python_class":{"python_class_name":"SqliteJobArchive","python_module_name":"kiara.registries.jobs.job_store.sqlite_store","full_name":"kiara.registries.jobs.job_store.sqlite_store.SqliteJobArchive"},"is_writable":false,"supported_item_types":["job_record"]},"config":{"sqlite_db_path":"/home/markus/projects/kiara/kiara/tests/resources/archives/nand_true.0.10.kiarchive","use_wal_mode":false},"details":{"no_job_records":4,"dynamic_archive":false},"metadata":{"archive_id":"1df47c50-058f-4f7a-8cdc-4ccab0ab1da9","archive_name":"nand_true.0.10"}}} \ No newline at end of file diff --git a/tests/test_archives/test_archive_export.py b/tests/test_archives/test_archive_export.py index 35909738d..39ec0435b 100644 --- a/tests/test_archives/test_archive_export.py +++ b/tests/test_archives/test_archive_export.py @@ -10,6 +10,16 @@ import pytest +from kiara.defaults import ( + ALL_REQUIRED_TABLES, + TABLE_NAME_DATA_PEDIGREE, + TABLE_NAME_DATA_METADATA, + TABLE_NAME_ARCHIVE_METADATA, + TABLE_NAME_DATA_CHUNKS, + TABLE_NAME_DATA_DESTINIES, + TABLE_NAME_DATA_SERIALIZATION_METADATA, + TABLE_NAME_ALIASES, +) from kiara.interfaces.python_api.base_api import BaseAPI from kiara.models.values.value import ValueMapReadOnly, Value @@ -115,27 +125,19 @@ def test_archive_export_values_no_alias(api: BaseAPI): assert len(store_result) == 1 assert "y" in store_result.keys() - required_tables = [ - "values_pedigree", - "values_metadata", - "archive_metadata", - "aliases", - "values_data", - "values_destinies", - "persisted_values", - ] - check_archive_contains_table_names(temp_file_path, required_tables) + required_tables = ALL_REQUIRED_TABLES - check_table_is_empty(temp_file_path, "aliases") + check_archive_contains_table_names(temp_file_path, required_tables) + check_table_is_empty(temp_file_path, TABLE_NAME_ALIASES) check_tables_are_not_empty( temp_file_path, - "values_pedigree", - "values_metadata", - "archive_metadata", - "values_data", - "values_destinies", - "persisted_values", + TABLE_NAME_DATA_PEDIGREE, + TABLE_NAME_DATA_METADATA, + TABLE_NAME_ARCHIVE_METADATA, + TABLE_NAME_DATA_CHUNKS, + TABLE_NAME_DATA_DESTINIES, + TABLE_NAME_DATA_SERIALIZATION_METADATA, ) @@ -167,29 +169,21 @@ def test_archive_export_values_alias(api: BaseAPI): assert len(store_result) == 1 assert "y" in store_result.keys() - required_tables = [ - "values_pedigree", - "values_metadata", - "archive_metadata", - "aliases", - "values_data", - "values_destinies", - "persisted_values", - ] + required_tables = ALL_REQUIRED_TABLES check_archive_contains_table_names(temp_file_path, required_tables) check_tables_are_not_empty( temp_file_path, - "values_pedigree", - "values_metadata", - "archive_metadata", - "values_data", - "values_destinies", - "persisted_values", - "aliases", + TABLE_NAME_DATA_PEDIGREE, + TABLE_NAME_DATA_METADATA, + TABLE_NAME_ARCHIVE_METADATA, + TABLE_NAME_DATA_CHUNKS, + TABLE_NAME_DATA_DESTINIES, + TABLE_NAME_DATA_SERIALIZATION_METADATA, + TABLE_NAME_ALIASES, ) - result = run_sql_query('SELECT * FROM "aliases";', temp_file_path) + result = run_sql_query(f'SELECT * FROM "{TABLE_NAME_ALIASES}";', temp_file_path) assert len(result) == 1 assert result[0][0] == "y" @@ -233,29 +227,21 @@ def test_archive_export_values_alias_multipe_values(api: BaseAPI): assert "result_1" in store_result.keys() assert "result_2" in store_result.keys() - required_tables = [ - "values_pedigree", - "values_metadata", - "archive_metadata", - "aliases", - "values_data", - "values_destinies", - "persisted_values", - ] + required_tables = ALL_REQUIRED_TABLES check_archive_contains_table_names(temp_file_path, required_tables) check_tables_are_not_empty( temp_file_path, - "values_pedigree", - "values_metadata", - "archive_metadata", - "values_data", - "values_destinies", - "persisted_values", - "aliases", + TABLE_NAME_DATA_PEDIGREE, + TABLE_NAME_DATA_METADATA, + TABLE_NAME_ARCHIVE_METADATA, + TABLE_NAME_DATA_CHUNKS, + TABLE_NAME_DATA_DESTINIES, + TABLE_NAME_DATA_SERIALIZATION_METADATA, + TABLE_NAME_ALIASES, ) - result = run_sql_query('SELECT * FROM "aliases";', temp_file_path) + result = run_sql_query(f'SELECT * FROM "{TABLE_NAME_ALIASES}";', temp_file_path) print(result) assert len(result) == 2 diff --git a/tests/test_archives/test_archive_import.py b/tests/test_archives/test_archive_import.py index 421d638ee..1f67bbb2c 100644 --- a/tests/test_archives/test_archive_import.py +++ b/tests/test_archives/test_archive_import.py @@ -8,7 +8,7 @@ ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) TEST_RESOURCES_FOLDER = os.path.join(ROOT_DIR, "tests", "resources") -VALUE_ID = "edbd6711-0432-430f-a147-08a6ae9df220" +VALUE_ID = "4c929a5b-c91a-449e-9d5f-f21124298ea7" def test_archive_import_values_no_alias(api: BaseAPI):