chore: first draft for metadata export

src/kiara/interfaces/cli/data/commands.py

@@ -15,6 +15,7 @@
 import rich_click as click
 import structlog
 
+from kiara.defaults import DATA_ARCHIVE_DEFAULT_VALUE_MARKER
 from kiara.exceptions import InvalidCommandLineInvocation
 from kiara.utils import is_develop, log_exception, log_message
 from kiara.utils.cli import output_format_option, terminal_print, terminal_print_model
@@ -645,6 +646,13 @@ def export_data_archive(
         "compression": compression,
     }
     try:
+        no_default_value = False
+        if not no_default_value:
+            metadata_to_add = {
+                DATA_ARCHIVE_DEFAULT_VALUE_MARKER: str(values[0][0].value_id)
+            }
+        else:
+            metadata_to_add = None
         store_result = kiara_api.export_values(
             target_archive=full_path,
             values=values_to_store,
@@ -653,6 +661,7 @@ def export_data_archive(
             target_registered_name=archive_name,
             append=append,
             target_store_params=target_store_params,
+            additional_archive_metadata=metadata_to_add,
         )
         render_config = {"add_field_column": False}
         terminal_print_model(

src/kiara/interfaces/python_api/__init__.py

@@ -2103,6 +2103,8 @@ def export_values(
         target_registered_name: Union[str, None] = None,
         append: bool = False,
         target_store_params: Union[None, Mapping[str, Any]] = None,
+        export_related_metadata: bool = True,
+        additional_archive_metadata: Union[None, Mapping[str, Any]] = None,
     ) -> StoreValuesResult:
         """Store one or several values along with (optional) aliases into a kiara archive.
 
@@ -2132,6 +2134,8 @@ def export_values(
             target_registered_name: the name to register the archive under in the context
             append: whether to append to an existing archive
             target_store_params: additional parameters to pass to the 'create_kiarchive' method if the file does not exist yet
+            export_related_metadata: whether to export related metadata (e.g. job info, comments, ..) to the new archive or not
+            additional_archive_metadata: (optional) additional metadata to add to the archive
 
         """
 
@@ -2158,6 +2162,14 @@ def export_values(
             allow_alias_overwrite=allow_alias_overwrite,
             store=target_archive_ref,
         )
+
+        if export_related_metadata:
+            raise NotImplementedError("xx")
+
+        if additional_archive_metadata:
+            for k, v in additional_archive_metadata.items():
+                self.set_archive_metadata_value(target_archive_ref, k, v)
+
         return result
 
     def register_archive(
@@ -2259,7 +2271,7 @@ def set_archive_metadata_value(
         archive: Union[str, uuid.UUID],
         key: str,
         value: Any,
-        archive_type: Literal["data", "alias"] = "data",
+        archive_type: Literal["data", "alias", "job_record", "metadata"] = "data",
     ) -> None:
         """Add metadata to an archive.
 
@@ -2279,6 +2291,16 @@ def set_archive_metadata_value(
             if _archive is None:
                 raise KiaraException(f"Archive '{archive}' does not exist.")
             _archive.set_archive_metadata_value(key, value)
+        elif archive_type == "metadata":
+            _archive = self.context.metadata_registry.get_archive(archive)
+            if _archive is None:
+                raise KiaraException(f"Archive '{archive}' does not exist.")
+            _archive.set_archive_metadata_value(key, value)
+        elif archive_type == "job_record":
+            _archive = self.context.job_registry.get_archive(archive)
+            if _archive is None:
+                raise KiaraException(f"Archive '{archive}' does not exist.")
+            _archive.set_archive_metadata_value(key, value)
         else:
             raise KiaraException(
                 f"Invalid archive type: {archive_type}. Valid types are: 'data', 'alias'."

src/kiara/interfaces/python_api/models/archive.py

@@ -7,11 +7,13 @@
 
 from kiara.defaults import CHUNK_COMPRESSION_TYPE
 from kiara.models import KiaraModel
+from kiara.registries.jobs import JobArchive
 
 if TYPE_CHECKING:
     from kiara.context import Kiara
     from kiara.registries.aliases import AliasArchive, AliasStore
     from kiara.registries.data import DataArchive, DataStore
+    from kiara.registries.jobs import JobArchive, JobStore
     from kiara.registries.metadata import MetadataArchive, MetadataStore
 
 
@@ -62,36 +64,40 @@ def load_kiarchive(
             alias_archive_config = None
             alias_archive = None
 
-        if data_archive is None and alias_archive is None:
-            raise Exception(f"No data archive found in file: {path}")
-        elif data_archive:
-            if alias_archive is not None:
-                if data_archive.archive_id != alias_archive.archive_id:
+        if "jobs_record" in archives.keys():
+            jobs_archive: Union[JobArchive, None] = archives["job_record"]  # type: ignore
+            jobs_archive_config: Union[Mapping[str, Any], None] = jobs_archive.config.model_dump()  # type: ignore
+        else:
+            jobs_archive_config = None
+            jobs_archive = None
+
+        archives = [
+            x
+            for x in (data_archive, alias_archive, metadata_archive, jobs_archive)
+            if x is not None
+        ]
+        if not archives:
+            raise Exception(f"No archive found in file: {path}")
+        else:
+            archive_id = archives[0].archive_id
+            archive_alias = archives[0].archive_alias
+            for archive in archives:
+                if archive.archive_id != archive_id:
                     raise Exception(
-                        f"Data and alias archives in file '{path}' have different IDs."
+                        f"Multiple different archive ids found in file: {path}"
                     )
-                if data_archive.archive_name != alias_archive.archive_name:
+                if archive.archive_alias != archive_alias:
                     raise Exception(
-                        f"Data and alias archives in file '{path}' have different aliases."
+                        f"Multiple different archive aliases found in file: {path}"
                     )
 
-            archive_id = data_archive.archive_id
-            archive_alias = data_archive.archive_name
-        elif alias_archive:
-            # we can assume data archive is None here
-            archive_id = alias_archive.archive_id
-            archive_alias = alias_archive.archive_name
-        else:
-            raise Exception(
-                "This should never happen, but we need to handle it anyway. Bug in code."
-            )
-
         kiarchive = KiArchive(
             archive_id=archive_id,
             archive_name=archive_alias,
             metadata_archive_config=metadata_archive_config,
             data_archive_config=data_archive_config,
             alias_archive_config=alias_archive_config,
+            jobs_archive_config=jobs_archive_config,
             archive_base_path=archive_path.parent.as_posix(),
             archive_file_name=archive_path.name,
             allow_write_access=allow_write_access,
@@ -154,6 +160,7 @@ def create_kiarchive(
                 store_type="sqlite_metadata_store",
                 file_name=archive_file_name,
                 allow_write_access=True,
+                set_archive_name_metadata=False,
             )
             metadata_store_config = metadata_store.config
 
@@ -163,12 +170,24 @@ def create_kiarchive(
                 store_type="sqlite_alias_store",
                 file_name=archive_file_name,
                 allow_write_access=allow_write_access,
+                set_archive_name_metadata=False,
             )
             alias_store_config = alias_store.config
 
+            job_store: JobStore = create_new_archive(
+                archive_name=archive_name,
+                store_base_path=archive_base_path,
+                store_type="sqlite_job_store",
+                file_name=archive_file_name,
+                allow_write_access=allow_write_access,
+                set_archive_name_metadata=False,
+            )
+            job_store_config = job_store.config
+
             kiarchive_id = data_store.archive_id
             assert alias_store.archive_id == kiarchive_id
             assert metadata_store.archive_id == kiarchive_id
+            assert job_store.archive_id == kiarchive_id
 
             kiarchive = KiArchive(
                 archive_id=kiarchive_id,
@@ -178,11 +197,13 @@ def create_kiarchive(
                 metadata_archive_config=metadata_store_config.model_dump(),
                 data_archive_config=data_store_config.model_dump(),
                 alias_archive_config=alias_store_config.model_dump(),
+                job_archive_config=job_store_config.model_dump(),
                 allow_write_access=allow_write_access,
             )
             kiarchive._metadata_archive = metadata_store
             kiarchive._data_archive = data_store
             kiarchive._alias_archive = alias_store
+            kiarchive._jobs_archive = job_store
             kiarchive._kiara = kiara
 
         return kiarchive
@@ -205,10 +226,15 @@ def create_kiarchive(
     alias_archive_config: Union[Mapping[str, Any], None] = Field(
         description="The archive to store aliases in.", default=None
     )
+    job_archive_config: Union[Mapping[str, Any], None] = Field(
+        description="The archive to store jobs in.", default=None
+    )
 
     _metadata_archive: Union["MetadataArchive", None] = PrivateAttr(default=None)
     _data_archive: Union["DataArchive", None] = PrivateAttr(default=None)
     _alias_archive: Union["AliasArchive", None] = PrivateAttr(default=None)
+    _jobs_archive: Union["JobArchive", None] = PrivateAttr(default=None)
+
     _kiara: Union["Kiara", None] = PrivateAttr(default=None)
 
     @property

src/kiara/registries/data/__init__.py

@@ -145,6 +145,7 @@ def resolve_alias(self, alias: str) -> uuid.UUID:
                         msg=f"Can't retrive value for alias '{rest}': no such alias registered.",
                     )
             elif ref_type == ARCHIVE_REF_TYPE_NAME:
+
                 if "#" in rest:
                     archive_ref, path_in_archive = rest.split("#", maxsplit=1)
                 else:
@@ -164,6 +165,10 @@ def resolve_alias(self, alias: str) -> uuid.UUID:
                             default_value = data_archive.get_archive_metadata(
                                 DATA_ARCHIVE_DEFAULT_VALUE_MARKER
                             )
+                            if default_value is None:
+                                raise NoSuchValueException(
+                                    f"No default value found for uri: {alias}"
+                                )
                             _value_id = uuid.UUID(default_value)
                         else:
                             from kiara.registries.aliases import AliasArchive
@@ -446,7 +451,11 @@ def get_value(self, value: Union[uuid.UUID, ValueLink, str, Path]) -> Value:
                         raise Exception(
                             f"Can't retrieve value for '{value}': invalid type '{type(value)}'."
                         )
-                    _value_id = self._alias_resolver.resolve_alias(value)
+                    try:
+                        _value_id = self._alias_resolver.resolve_alias(value)
+                    except Exception as e:
+                        log_exception(e)
+                        raise e
         else:
             _value_id = value
 
@@ -1301,8 +1310,8 @@ def create_valuemap(
                     else:
                         try:
                             _d = self._alias_resolver.resolve_alias(_d)
-                        except Exception:
-                            pass
+                        except Exception as e:
+                            log_exception(e)
 
             if isinstance(_d, Value):
                 _resolved[input_name] = _d

src/kiara/registries/jobs/job_store/sqlite_store.py

@@ -329,3 +329,14 @@ def store_job_record(self, job_record: JobRecord):
             connection.execute(sql, params)
 
             connection.commit()
+
+    def _set_archive_metadata_value(self, key: str, value: Any):
+        """Set custom metadata for the archive."""
+
+        sql = text(
+            "INSERT OR REPLACE INTO archive_metadata (key, value) VALUES (:key, :value)"
+        )
+        with self.sqlite_engine.connect() as conn:
+            params = {"key": key, "value": value}
+            conn.execute(sql, params)
+            conn.commit()

src/kiara/utils/stores.py

@@ -12,8 +12,19 @@ def create_new_archive(
     store_base_path: str,
     store_type: str,
     allow_write_access: bool = False,
+    set_archive_name_metadata: bool = True,
     **kwargs: Any,
 ) -> "KiaraArchive":
+    """Create a new archive instance of the specified type.
+
+    Arguments:
+        archive_name: Name of the archive.
+        store_base_path: Base path for the archive.
+        store_type: Type of the archive.
+        allow_write_access: Whether write access should be allowed.
+        set_archive_name_metadata: Whether to set the archive name as metadata within the archive.
+        **kwargs: Additional arguments to pass to the archive config constructor.
+    """
 
     from kiara.utils.class_loading import find_all_archive_types
 
@@ -33,7 +44,7 @@ def create_new_archive(
 
     archive_instance = archive_cls(archive_name=archive_name, archive_config=config, force_read_only=force_read_only)  # type: ignore
 
-    if not force_read_only:
+    if not force_read_only and set_archive_name_metadata:
         archive_instance.set_archive_metadata_value(ARCHIVE_NAME_MARKER, archive_name)
 
     return archive_instance