diff --git a/src/kiara/interfaces/cli/data/commands.py b/src/kiara/interfaces/cli/data/commands.py index b6f7e301d..3ce2d409a 100644 --- a/src/kiara/interfaces/cli/data/commands.py +++ b/src/kiara/interfaces/cli/data/commands.py @@ -15,6 +15,7 @@ import rich_click as click import structlog +from kiara.defaults import DATA_ARCHIVE_DEFAULT_VALUE_MARKER from kiara.exceptions import InvalidCommandLineInvocation from kiara.utils import is_develop, log_exception, log_message from kiara.utils.cli import output_format_option, terminal_print, terminal_print_model @@ -645,6 +646,13 @@ def export_data_archive( "compression": compression, } try: + no_default_value = False + if not no_default_value: + metadata_to_add = { + DATA_ARCHIVE_DEFAULT_VALUE_MARKER: str(values[0][0].value_id) + } + else: + metadata_to_add = None store_result = kiara_api.export_values( target_archive=full_path, values=values_to_store, @@ -653,6 +661,7 @@ def export_data_archive( target_registered_name=archive_name, append=append, target_store_params=target_store_params, + additional_archive_metadata=metadata_to_add, ) render_config = {"add_field_column": False} terminal_print_model( diff --git a/src/kiara/interfaces/python_api/__init__.py b/src/kiara/interfaces/python_api/__init__.py index f9691260a..82bc900a6 100644 --- a/src/kiara/interfaces/python_api/__init__.py +++ b/src/kiara/interfaces/python_api/__init__.py @@ -2103,6 +2103,8 @@ def export_values( target_registered_name: Union[str, None] = None, append: bool = False, target_store_params: Union[None, Mapping[str, Any]] = None, + export_related_metadata: bool = True, + additional_archive_metadata: Union[None, Mapping[str, Any]] = None, ) -> StoreValuesResult: """Store one or several values along with (optional) aliases into a kiara archive. @@ -2132,6 +2134,8 @@ def export_values( target_registered_name: the name to register the archive under in the context append: whether to append to an existing archive target_store_params: additional parameters to pass to the 'create_kiarchive' method if the file does not exist yet + export_related_metadata: whether to export related metadata (e.g. job info, comments, ..) to the new archive or not + additional_archive_metadata: (optional) additional metadata to add to the archive """ @@ -2158,6 +2162,14 @@ def export_values( allow_alias_overwrite=allow_alias_overwrite, store=target_archive_ref, ) + + if export_related_metadata: + raise NotImplementedError("xx") + + if additional_archive_metadata: + for k, v in additional_archive_metadata.items(): + self.set_archive_metadata_value(target_archive_ref, k, v) + return result def register_archive( @@ -2259,7 +2271,7 @@ def set_archive_metadata_value( archive: Union[str, uuid.UUID], key: str, value: Any, - archive_type: Literal["data", "alias"] = "data", + archive_type: Literal["data", "alias", "job_record", "metadata"] = "data", ) -> None: """Add metadata to an archive. @@ -2279,6 +2291,16 @@ def set_archive_metadata_value( if _archive is None: raise KiaraException(f"Archive '{archive}' does not exist.") _archive.set_archive_metadata_value(key, value) + elif archive_type == "metadata": + _archive = self.context.metadata_registry.get_archive(archive) + if _archive is None: + raise KiaraException(f"Archive '{archive}' does not exist.") + _archive.set_archive_metadata_value(key, value) + elif archive_type == "job_record": + _archive = self.context.job_registry.get_archive(archive) + if _archive is None: + raise KiaraException(f"Archive '{archive}' does not exist.") + _archive.set_archive_metadata_value(key, value) else: raise KiaraException( f"Invalid archive type: {archive_type}. Valid types are: 'data', 'alias'." diff --git a/src/kiara/interfaces/python_api/models/archive.py b/src/kiara/interfaces/python_api/models/archive.py index 5f964a1de..ff5a6d960 100644 --- a/src/kiara/interfaces/python_api/models/archive.py +++ b/src/kiara/interfaces/python_api/models/archive.py @@ -7,11 +7,13 @@ from kiara.defaults import CHUNK_COMPRESSION_TYPE from kiara.models import KiaraModel +from kiara.registries.jobs import JobArchive if TYPE_CHECKING: from kiara.context import Kiara from kiara.registries.aliases import AliasArchive, AliasStore from kiara.registries.data import DataArchive, DataStore + from kiara.registries.jobs import JobArchive, JobStore from kiara.registries.metadata import MetadataArchive, MetadataStore @@ -62,36 +64,40 @@ def load_kiarchive( alias_archive_config = None alias_archive = None - if data_archive is None and alias_archive is None: - raise Exception(f"No data archive found in file: {path}") - elif data_archive: - if alias_archive is not None: - if data_archive.archive_id != alias_archive.archive_id: + if "jobs_record" in archives.keys(): + jobs_archive: Union[JobArchive, None] = archives["job_record"] # type: ignore + jobs_archive_config: Union[Mapping[str, Any], None] = jobs_archive.config.model_dump() # type: ignore + else: + jobs_archive_config = None + jobs_archive = None + + archives = [ + x + for x in (data_archive, alias_archive, metadata_archive, jobs_archive) + if x is not None + ] + if not archives: + raise Exception(f"No archive found in file: {path}") + else: + archive_id = archives[0].archive_id + archive_alias = archives[0].archive_alias + for archive in archives: + if archive.archive_id != archive_id: raise Exception( - f"Data and alias archives in file '{path}' have different IDs." + f"Multiple different archive ids found in file: {path}" ) - if data_archive.archive_name != alias_archive.archive_name: + if archive.archive_alias != archive_alias: raise Exception( - f"Data and alias archives in file '{path}' have different aliases." + f"Multiple different archive aliases found in file: {path}" ) - archive_id = data_archive.archive_id - archive_alias = data_archive.archive_name - elif alias_archive: - # we can assume data archive is None here - archive_id = alias_archive.archive_id - archive_alias = alias_archive.archive_name - else: - raise Exception( - "This should never happen, but we need to handle it anyway. Bug in code." - ) - kiarchive = KiArchive( archive_id=archive_id, archive_name=archive_alias, metadata_archive_config=metadata_archive_config, data_archive_config=data_archive_config, alias_archive_config=alias_archive_config, + jobs_archive_config=jobs_archive_config, archive_base_path=archive_path.parent.as_posix(), archive_file_name=archive_path.name, allow_write_access=allow_write_access, @@ -154,6 +160,7 @@ def create_kiarchive( store_type="sqlite_metadata_store", file_name=archive_file_name, allow_write_access=True, + set_archive_name_metadata=False, ) metadata_store_config = metadata_store.config @@ -163,12 +170,24 @@ def create_kiarchive( store_type="sqlite_alias_store", file_name=archive_file_name, allow_write_access=allow_write_access, + set_archive_name_metadata=False, ) alias_store_config = alias_store.config + job_store: JobStore = create_new_archive( + archive_name=archive_name, + store_base_path=archive_base_path, + store_type="sqlite_job_store", + file_name=archive_file_name, + allow_write_access=allow_write_access, + set_archive_name_metadata=False, + ) + job_store_config = job_store.config + kiarchive_id = data_store.archive_id assert alias_store.archive_id == kiarchive_id assert metadata_store.archive_id == kiarchive_id + assert job_store.archive_id == kiarchive_id kiarchive = KiArchive( archive_id=kiarchive_id, @@ -178,11 +197,13 @@ def create_kiarchive( metadata_archive_config=metadata_store_config.model_dump(), data_archive_config=data_store_config.model_dump(), alias_archive_config=alias_store_config.model_dump(), + job_archive_config=job_store_config.model_dump(), allow_write_access=allow_write_access, ) kiarchive._metadata_archive = metadata_store kiarchive._data_archive = data_store kiarchive._alias_archive = alias_store + kiarchive._jobs_archive = job_store kiarchive._kiara = kiara return kiarchive @@ -205,10 +226,15 @@ def create_kiarchive( alias_archive_config: Union[Mapping[str, Any], None] = Field( description="The archive to store aliases in.", default=None ) + job_archive_config: Union[Mapping[str, Any], None] = Field( + description="The archive to store jobs in.", default=None + ) _metadata_archive: Union["MetadataArchive", None] = PrivateAttr(default=None) _data_archive: Union["DataArchive", None] = PrivateAttr(default=None) _alias_archive: Union["AliasArchive", None] = PrivateAttr(default=None) + _jobs_archive: Union["JobArchive", None] = PrivateAttr(default=None) + _kiara: Union["Kiara", None] = PrivateAttr(default=None) @property diff --git a/src/kiara/registries/data/__init__.py b/src/kiara/registries/data/__init__.py index fff8cd988..62456592f 100644 --- a/src/kiara/registries/data/__init__.py +++ b/src/kiara/registries/data/__init__.py @@ -145,6 +145,7 @@ def resolve_alias(self, alias: str) -> uuid.UUID: msg=f"Can't retrive value for alias '{rest}': no such alias registered.", ) elif ref_type == ARCHIVE_REF_TYPE_NAME: + if "#" in rest: archive_ref, path_in_archive = rest.split("#", maxsplit=1) else: @@ -164,6 +165,10 @@ def resolve_alias(self, alias: str) -> uuid.UUID: default_value = data_archive.get_archive_metadata( DATA_ARCHIVE_DEFAULT_VALUE_MARKER ) + if default_value is None: + raise NoSuchValueException( + f"No default value found for uri: {alias}" + ) _value_id = uuid.UUID(default_value) else: from kiara.registries.aliases import AliasArchive @@ -446,7 +451,11 @@ def get_value(self, value: Union[uuid.UUID, ValueLink, str, Path]) -> Value: raise Exception( f"Can't retrieve value for '{value}': invalid type '{type(value)}'." ) - _value_id = self._alias_resolver.resolve_alias(value) + try: + _value_id = self._alias_resolver.resolve_alias(value) + except Exception as e: + log_exception(e) + raise e else: _value_id = value @@ -1301,8 +1310,8 @@ def create_valuemap( else: try: _d = self._alias_resolver.resolve_alias(_d) - except Exception: - pass + except Exception as e: + log_exception(e) if isinstance(_d, Value): _resolved[input_name] = _d diff --git a/src/kiara/registries/jobs/job_store/sqlite_store.py b/src/kiara/registries/jobs/job_store/sqlite_store.py index b6bff4e3f..e5c58883c 100644 --- a/src/kiara/registries/jobs/job_store/sqlite_store.py +++ b/src/kiara/registries/jobs/job_store/sqlite_store.py @@ -329,3 +329,14 @@ def store_job_record(self, job_record: JobRecord): connection.execute(sql, params) connection.commit() + + def _set_archive_metadata_value(self, key: str, value: Any): + """Set custom metadata for the archive.""" + + sql = text( + "INSERT OR REPLACE INTO archive_metadata (key, value) VALUES (:key, :value)" + ) + with self.sqlite_engine.connect() as conn: + params = {"key": key, "value": value} + conn.execute(sql, params) + conn.commit() diff --git a/src/kiara/utils/stores.py b/src/kiara/utils/stores.py index 0a79628b9..b8c4bd55d 100644 --- a/src/kiara/utils/stores.py +++ b/src/kiara/utils/stores.py @@ -12,8 +12,19 @@ def create_new_archive( store_base_path: str, store_type: str, allow_write_access: bool = False, + set_archive_name_metadata: bool = True, **kwargs: Any, ) -> "KiaraArchive": + """Create a new archive instance of the specified type. + + Arguments: + archive_name: Name of the archive. + store_base_path: Base path for the archive. + store_type: Type of the archive. + allow_write_access: Whether write access should be allowed. + set_archive_name_metadata: Whether to set the archive name as metadata within the archive. + **kwargs: Additional arguments to pass to the archive config constructor. + """ from kiara.utils.class_loading import find_all_archive_types @@ -33,7 +44,7 @@ def create_new_archive( archive_instance = archive_cls(archive_name=archive_name, archive_config=config, force_read_only=force_read_only) # type: ignore - if not force_read_only: + if not force_read_only and set_archive_name_metadata: archive_instance.set_archive_metadata_value(ARCHIVE_NAME_MARKER, archive_name) return archive_instance