From c367775b3d24c8173820cbe6430b17bd81bcf195 Mon Sep 17 00:00:00 2001 From: Markus Binsteiner Date: Thu, 30 Nov 2023 16:25:35 +0100 Subject: [PATCH] feat: add a way to specify operations that are attached to modules --- src/kiara/interfaces/cli/info/commands.py | 3 +- src/kiara/interfaces/cli/run.py | 9 ++ .../interfaces/python_api/models/info.py | 2 +- src/kiara/interfaces/python_api/workflow.py | 5 +- src/kiara/models/__init__.py | 7 +- src/kiara/models/module/jobs.py | 13 +- src/kiara/models/module/manifest.py | 25 ++-- src/kiara/models/module/operation.py | 15 ++ src/kiara/models/values/value.py | 6 +- src/kiara/modules/__init__.py | 2 + .../included_core_operations/__init__.py | 128 ++++++++++++++---- .../included_core_operations/create_from.py | 4 +- .../included_core_operations/export_as.py | 4 +- .../included_core_operations/filter.py | 4 +- .../included_core_operations/import_data.py | 4 +- .../included_core_operations/metadata.py | 4 +- .../included_core_operations/pipeline.py | 4 +- .../included_core_operations/pretty_print.py | 4 +- .../included_core_operations/render_data.py | 4 +- .../included_core_operations/render_value.py | 4 +- .../included_core_operations/serialize.py | 4 +- src/kiara/registries/data/__init__.py | 3 + src/kiara/registries/modules/__init__.py | 53 +++++--- src/kiara/registries/operations/__init__.py | 9 +- src/kiara/utils/cli/run.py | 28 +++- 25 files changed, 256 insertions(+), 92 deletions(-) diff --git a/src/kiara/interfaces/cli/info/commands.py b/src/kiara/interfaces/cli/info/commands.py index a79098c04..77d7e152e 100644 --- a/src/kiara/interfaces/cli/info/commands.py +++ b/src/kiara/interfaces/cli/info/commands.py @@ -7,7 +7,6 @@ # Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) import rich_click as click -from kiara.interfaces.python_api import KiaraPluginInfos from kiara.utils.cli import output_format_option, terminal_print_model from kiara.utils.cli.exceptions import handle_exception @@ -56,6 +55,8 @@ def plugin(ctx): def list_plugins(ctx, filter_regex: str, format): """List installed kiara plugins.""" + from kiara.interfaces.python_api import KiaraPluginInfos + api: KiaraAPI = ctx.obj.kiara_api title = "All available plugins" diff --git a/src/kiara/interfaces/cli/run.py b/src/kiara/interfaces/cli/run.py index 26e01213e..f5760426b 100644 --- a/src/kiara/interfaces/cli/run.py +++ b/src/kiara/interfaces/cli/run.py @@ -51,6 +51,13 @@ required=False, multiple=True, ) +@click.option( + "--print-properties", + "-p", + help="Also display the properties of the result values.", + required=False, + is_flag=True, +) @click.option("--help", "-h", help="Show this message and exit.", is_flag=True) @click.pass_context @handle_exception() @@ -62,6 +69,7 @@ def run( output: Iterable[str], explain: bool, save: Iterable[str], + print_properties: bool, help: bool, ): """Run a kiara operation.""" @@ -244,4 +252,5 @@ def run( silent=silent, save_results=bool(final_aliases), aliases=final_aliases, + properties=print_properties, ) diff --git a/src/kiara/interfaces/python_api/models/info.py b/src/kiara/interfaces/python_api/models/info.py index ea69b029c..221a95719 100644 --- a/src/kiara/interfaces/python_api/models/info.py +++ b/src/kiara/interfaces/python_api/models/info.py @@ -1047,7 +1047,7 @@ def extract_module_attributes( def create_renderable(self, **config: Any) -> RenderableType: include_config_schema = config.get("include_config_schema", True) - include_src = config.get("include_src", True) + include_src = config.get("include_src", False) include_doc = config.get("include_doc", True) table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0)) diff --git a/src/kiara/interfaces/python_api/workflow.py b/src/kiara/interfaces/python_api/workflow.py index e49e2f4dd..e14bd6269 100644 --- a/src/kiara/interfaces/python_api/workflow.py +++ b/src/kiara/interfaces/python_api/workflow.py @@ -656,8 +656,9 @@ def _apply_inputs(self) -> Mapping[str, Mapping[str, Mapping[str, ChangedValue]] break elif step_details.status == StepStatus.INPUTS_READY: job_config = JobConfig( - module_type=step_details.step.module_type, - module_config=step_details.step.module.config.model_dump(), + module_type=step_details.step.module.manifest.module_type, + module_config=step_details.step.module.manifest.module_config, + is_resolved=step_details.step.module.manifest.is_resolved, inputs=step_details.inputs, ) match = self._kiara.job_registry.find_matching_job_record( diff --git a/src/kiara/models/__init__.py b/src/kiara/models/__init__.py index fa29931a0..8b5df392d 100644 --- a/src/kiara/models/__init__.py +++ b/src/kiara/models/__init__.py @@ -28,6 +28,7 @@ KIARA_MODEL_SCHEMA_KEY, ) from kiara.registries.templates import TemplateRegistry +from kiara.utils import log_exception from kiara.utils.class_loading import _default_id_func from kiara.utils.develop import log_dev_message from kiara.utils.hashing import KIARA_HASH_FUNCTION, compute_cid @@ -133,7 +134,11 @@ def _compute_cid(self): return obj = self._retrieve_data_to_hash() - dag, cid = compute_cid(data=obj) + try: + dag, cid = compute_cid(data=obj) + except Exception as e: + log_exception(e) + raise e self._cid_cache = cid self._dag_cache = dag diff --git a/src/kiara/models/module/jobs.py b/src/kiara/models/module/jobs.py index 775c376f7..7a3cb9f1c 100644 --- a/src/kiara/models/module/jobs.py +++ b/src/kiara/models/module/jobs.py @@ -19,7 +19,7 @@ from rich.console import RenderableType from rich.table import Table -from kiara.exceptions import InvalidValuesException +from kiara.exceptions import InvalidValuesException, KiaraException from kiara.models import KiaraModel from kiara.models.module.manifest import InputsManifest @@ -97,9 +97,16 @@ def create_from_module( raise InvalidValuesException(invalid_values=invalid) value_ids = values.get_all_value_ids() + + if not module.manifest.is_resolved: + raise KiaraException( + msg="Cannot create job config from unresolved manifest." + ) + return JobConfig( - module_type=module.module_type_name, - module_config=module.config.model_dump(), + module_type=module.manifest.module_type, + module_config=module.manifest.module_config, + is_resolved=module.manifest.is_resolved, inputs=value_ids, ) diff --git a/src/kiara/models/module/manifest.py b/src/kiara/models/module/manifest.py index f409f1e91..43e3056b2 100644 --- a/src/kiara/models/module/manifest.py +++ b/src/kiara/models/module/manifest.py @@ -11,11 +11,12 @@ import orjson from dag_cbor import IPLDKind from multiformats import CID -from pydantic import ConfigDict, Field, PrivateAttr, field_validator +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, field_validator from rich.console import RenderableType from rich.syntax import Syntax from kiara.defaults import INVALID_HASH_MARKER, NONE_VALUE_ID +from kiara.exceptions import KiaraException from kiara.models import KiaraModel from kiara.utils.hashing import compute_cid from kiara.utils.json import orjson_dumps @@ -53,6 +54,14 @@ class Manifest(KiaraModel): # # return value + @field_validator("module_config") + @classmethod + def validate_module_config(cls, value): + if isinstance(value, BaseModel): + raise ValueError(f"Invalid module config type: {type(value)}") + + return value + @property def manifest_data(self): """The configuration data for this module instance.""" @@ -60,6 +69,7 @@ def manifest_data(self): return self._manifest_data mc = extract_data_to_hash_from_pipeline_config(self.module_config) + self._manifest_data = { "module_type": self.module_type, "module_config": mc, @@ -72,6 +82,11 @@ def manifest_cid(self) -> CID: if self._manifest_cid is not None: return self._manifest_cid + if not self.is_resolved: + raise KiaraException( + msg="Cannot calculate manifest CID for unresolved manifest." + ) + _, self._manifest_cid = compute_cid(self.manifest_data) return self._manifest_cid @@ -85,13 +100,7 @@ def manifest_data_as_json(self): def _retrieve_data_to_hash(self) -> Any: - module_config = extract_data_to_hash_from_pipeline_config(self.module_config) - result = { - "module_type": self.module_type, - "module_config": module_config, - } - - return result + return self.manifest_data def create_renderable(self, **config: Any) -> RenderableType: """Create a renderable for this module configuration.""" diff --git a/src/kiara/models/module/operation.py b/src/kiara/models/module/operation.py index 902e51eb3..8b03582ce 100644 --- a/src/kiara/models/module/operation.py +++ b/src/kiara/models/module/operation.py @@ -161,12 +161,27 @@ class ManifestOperationConfig(OperationConfig): default_factory=dict, description="The configuration for the module." ) + _manifest_cache: Union[None, Manifest] = PrivateAttr(default=None) + + @field_validator("doc", mode="before") + @classmethod + def validate_doc(cls, value): + return DocumentationMetadataModel.create(value) + def retrieve_module_type(self, kiara: "Kiara") -> str: return self.module_type def retrieve_module_config(self, kiara: "Kiara") -> Mapping[str, Any]: return self.module_config + def get_manifest(self) -> Manifest: + + if self._manifest_cache is None: + self._manifest_cache = Manifest( + module_type=self.module_type, module_config=self.module_config + ) + return self._manifest_cache + class PipelineOperationConfig(OperationConfig): diff --git a/src/kiara/models/values/value.py b/src/kiara/models/values/value.py index 1be28b605..a4a806099 100644 --- a/src/kiara/models/values/value.py +++ b/src/kiara/models/values/value.py @@ -1648,6 +1648,10 @@ def set_value(self, field_name: str, data: Any) -> None: ValuePedigree.model_rebuild() ORPHAN = ValuePedigree( - kiara_id=VOID_KIARA_ID, environments={}, module_type=NO_MODULE_TYPE, inputs={} + kiara_id=VOID_KIARA_ID, + environments={}, + module_type=NO_MODULE_TYPE, + inputs={}, + is_resolved=True, ) # GENESIS_PEDIGREE = None diff --git a/src/kiara/modules/__init__.py b/src/kiara/modules/__init__.py index f18229f78..0b66bf9b9 100644 --- a/src/kiara/modules/__init__.py +++ b/src/kiara/modules/__init__.py @@ -395,6 +395,8 @@ def __init__( @property def manifest(self) -> "Manifest": if self._manifest_cache is None: + from kiara.models.module.manifest import Manifest + self._manifest_cache = Manifest( module_type=self.module_type_name, module_config=self.config.model_dump(), diff --git a/src/kiara/operations/included_core_operations/__init__.py b/src/kiara/operations/included_core_operations/__init__.py index 8e0867888..2213b54b7 100644 --- a/src/kiara/operations/included_core_operations/__init__.py +++ b/src/kiara/operations/included_core_operations/__init__.py @@ -5,11 +5,12 @@ # # Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) -from typing import TYPE_CHECKING, Iterable, Mapping, Union +from typing import TYPE_CHECKING, ClassVar, Dict, Iterable, List, Mapping, Type, Union import structlog from pydantic import Field, PrivateAttr +from kiara.exceptions import KiaraException from kiara.models.documentation import DocumentationMetadataModel from kiara.models.module.operation import ( ManifestOperationConfig, @@ -22,8 +23,9 @@ from kiara.operations import OperationType if TYPE_CHECKING: - pass + from multiformats import CID + from kiara.context import Kiara logger = structlog.getLogger() @@ -61,7 +63,78 @@ def get_operation_schema(self) -> OperationSchema: class CustomModuleOperationType(OperationType[CustomModuleOperationDetails]): - _operation_type_name = "custom_module" + _operation_type_name: ClassVar[str] = "custom_module" + + def __init__(self, kiara: "Kiara", op_type_name: str): + self._included_operations_cache: Dict[ + type, List["ManifestOperationConfig"] + ] = {} + self._included_operations_lookup_cache: Dict[type, Dict["CID", str]] = {} + + super().__init__(kiara=kiara, op_type_name=op_type_name) + + def _retrieve_included_operations( + self, module_cls: Type[KiaraModule] + ) -> List["ManifestOperationConfig"]: + + if self._included_operations_cache.get(module_cls, None) is not None: + return self._included_operations_cache.get(module_cls) # type: ignore + + from kiara.models.module.operation import ManifestOperationConfig + + this_module_type_name: str = module_cls._module_type_name # type: ignore + + doc = None + cache: List[ManifestOperationConfig] = [] + lookup_cache: Dict["CID", str] = {} + + if not module_cls._config_cls.requires_config(): + doc = DocumentationMetadataModel.from_class_doc(module_cls) + mopc = ManifestOperationConfig(module_type=this_module_type_name, doc=doc) + resolved = self._kiara.module_registry.resolve_manifest(mopc.get_manifest()) + mopc._manifest_cache = resolved + + cache.append(mopc) + lookup_cache[resolved.manifest_cid] = this_module_type_name + + if hasattr(module_cls, "retrieve_included_operations"): + manifests = module_cls.retrieve_included_operations() # type: ignore + for op_id, op in manifests.items(): + if isinstance(op, Mapping): + mtn = op.get("module_type", None) + if not mtn: + op = dict(op) + op["module_type"] = this_module_type_name + if "doc" not in op.keys(): + if doc is None: + doc = DocumentationMetadataModel.from_class_doc( + module_cls + ) + op["doc"] = doc + elif not mtn != this_module_type_name: + raise KiaraException( + msg=f"Included operation '{op_id}' invalid.", + reason=f"module_type must be empty or set to the name '{this_module_type_name}'.", + ) + mopc = ManifestOperationConfig(**op) + elif not isinstance(op, ManifestOperationConfig): + raise KiaraException( + msg=f"Included operation '{op_id}' invalid.", + reason="Must be a Mapping or ManifestOperationConfig instance.", + ) + else: + mopc = op + + cache.append(mopc) + resolved = self._kiara.module_registry.resolve_manifest( + mopc.get_manifest() + ) + mopc._manifest_cache = resolved + lookup_cache[resolved.manifest_cid] = op_id + + self._included_operations_cache[module_cls] = cache + self._included_operations_lookup_cache[module_cls] = lookup_cache + return self._included_operations_cache[module_cls] def retrieve_included_operation_configs( self, @@ -69,37 +142,34 @@ def retrieve_included_operation_configs( result = [] for name, module_cls in self._kiara.module_type_classes.items(): - mod_conf = module_cls._config_cls + configs = self._retrieve_included_operations(module_cls=module_cls) + result.extend(configs) - if mod_conf.requires_config(): - logger.debug( - "ignore.custom_operation", - module_type=name, - reason="config required", - ) - continue - doc = DocumentationMetadataModel.from_class_doc(module_cls) - oc = ManifestOperationConfig(module_type=name, doc=doc) - result.append(oc) return result def check_matching_operation( self, module: "KiaraModule" ) -> Union[CustomModuleOperationDetails, None]: - mod_conf = module.__class__._config_cls - - if not mod_conf.requires_config(): - is_internal = module.characteristics.is_internal - # inputs_map = {k: k for k in module.inputs_schema.keys()} - # outputs_map = {k: k for k in module.outputs_schema.keys()} - op_details: CustomModuleOperationDetails = ( - CustomModuleOperationDetails.create_operation_details( - operation_id=module.module_type_name, - module_inputs_schema=module.inputs_schema, - module_outputs_schema=module.outputs_schema, - is_internal_operation=is_internal, - ) + + op_id: Union[str, None] = None + if not module.is_pipeline(): + manifest_cid = module.manifest.manifest_cid + op_id = self._included_operations_lookup_cache[module.__class__].get( + manifest_cid ) - return op_details - else: + + if not op_id: return None + + is_internal = module.characteristics.is_internal + # inputs_map = {k: k for k in module.inputs_schema.keys()} + # outputs_map = {k: k for k in module.outputs_schema.keys()} + op_details: CustomModuleOperationDetails = ( + CustomModuleOperationDetails.create_operation_details( + operation_id=op_id, + module_inputs_schema=module.inputs_schema, + module_outputs_schema=module.outputs_schema, + is_internal_operation=is_internal, + ) + ) + return op_details diff --git a/src/kiara/operations/included_core_operations/create_from.py b/src/kiara/operations/included_core_operations/create_from.py index 5260e06d7..8c7d19d39 100644 --- a/src/kiara/operations/included_core_operations/create_from.py +++ b/src/kiara/operations/included_core_operations/create_from.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from typing import TYPE_CHECKING, Iterable, Mapping, Union +from typing import TYPE_CHECKING, ClassVar, Iterable, Mapping, Union import structlog from pydantic import Field @@ -54,7 +54,7 @@ class CreateFromOperationType(OperationType[CreateValueFromDetails]): in addition, also have other, optional inputs, to control how exactly the target value is created. """ - _operation_type_name = "create_from" + _operation_type_name: ClassVar[str] = "create_from" def _calculate_op_id(self, source_type: str, target_type: str): diff --git a/src/kiara/operations/included_core_operations/export_as.py b/src/kiara/operations/included_core_operations/export_as.py index 3452f26e4..e3f07b363 100644 --- a/src/kiara/operations/included_core_operations/export_as.py +++ b/src/kiara/operations/included_core_operations/export_as.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from typing import TYPE_CHECKING, Iterable, Mapping, Union +from typing import TYPE_CHECKING, ClassVar, Iterable, Mapping, Union import structlog from pydantic import Field @@ -52,7 +52,7 @@ class ExportAsOperationDetails(BaseOperationDetails): class ExportAsOperationType(OperationType[ExportAsOperationDetails]): - _operation_type_name = "export_as" + _operation_type_name: ClassVar[str] = "export_as" def _calculate_op_id(self, source_type: str, target_profile: str): diff --git a/src/kiara/operations/included_core_operations/filter.py b/src/kiara/operations/included_core_operations/filter.py index 89219df80..46fdec4cd 100644 --- a/src/kiara/operations/included_core_operations/filter.py +++ b/src/kiara/operations/included_core_operations/filter.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Mapping, Union +from typing import TYPE_CHECKING, Any, ClassVar, Dict, Iterable, List, Mapping, Union import structlog from pydantic import Field @@ -84,7 +84,7 @@ class FilterOperationDetails(BaseOperationDetails): class FilterOperationType(OperationType[FilterOperationDetails]): - _operation_type_name = "filter" + _operation_type_name: ClassVar[str] = "filter" def retrieve_included_operation_configs( self, diff --git a/src/kiara/operations/included_core_operations/import_data.py b/src/kiara/operations/included_core_operations/import_data.py index 089176529..7e97298a2 100644 --- a/src/kiara/operations/included_core_operations/import_data.py +++ b/src/kiara/operations/included_core_operations/import_data.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from typing import TYPE_CHECKING, Iterable, Mapping, Union +from typing import TYPE_CHECKING, ClassVar, Iterable, Mapping, Union import structlog from pydantic import Field @@ -45,7 +45,7 @@ class ImportDataOpDetails(BaseOperationDetails): class ImportDataOperationType(OperationType[ImportDataOpDetails]): - _operation_type_name = "import_data" + _operation_type_name: ClassVar[str] = "import_data" def _calculate_op_id(self, source_type: str, target_type: str): diff --git a/src/kiara/operations/included_core_operations/metadata.py b/src/kiara/operations/included_core_operations/metadata.py index a9faaa600..80ee8a44e 100644 --- a/src/kiara/operations/included_core_operations/metadata.py +++ b/src/kiara/operations/included_core_operations/metadata.py @@ -5,7 +5,7 @@ # # Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) -from typing import Iterable, Mapping, Type, Union +from typing import ClassVar, Iterable, Mapping, Type, Union from pydantic import Field @@ -62,7 +62,7 @@ class ExtractMetadataOperationType(OperationType[ExtractMetadataDetails]): - exactly one output field, whose field name is called 'value_metadata', and where the value has the type 'internal_model' """ - _operation_type_name = "extract_metadata" + _operation_type_name: ClassVar[str] = "extract_metadata" def retrieve_included_operation_configs( self, diff --git a/src/kiara/operations/included_core_operations/pipeline.py b/src/kiara/operations/included_core_operations/pipeline.py index edefbb88d..d78083a12 100644 --- a/src/kiara/operations/included_core_operations/pipeline.py +++ b/src/kiara/operations/included_core_operations/pipeline.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- import os -from typing import TYPE_CHECKING, Any, Dict, Iterable, Mapping, Union +from typing import TYPE_CHECKING, Any, ClassVar, Dict, Iterable, Mapping, Union import structlog from pydantic import Field, PrivateAttr @@ -65,7 +65,7 @@ def get_operation_schema(self) -> OperationSchema: class PipelineOperationType(OperationType[PipelineOperationDetails]): - _operation_type_name = "pipeline" + _operation_type_name: ClassVar[str] = "pipeline" def __init__(self, kiara: "Kiara", op_type_name: str) -> None: diff --git a/src/kiara/operations/included_core_operations/pretty_print.py b/src/kiara/operations/included_core_operations/pretty_print.py index 2e994b45a..366aa903e 100644 --- a/src/kiara/operations/included_core_operations/pretty_print.py +++ b/src/kiara/operations/included_core_operations/pretty_print.py @@ -5,7 +5,7 @@ # # Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) -from typing import Dict, Iterable, Mapping, Union +from typing import ClassVar, Dict, Iterable, Mapping, Union from pydantic import Field @@ -67,7 +67,7 @@ class PrettyPrintOperationType(OperationType[PrettyPrintDetails]): - exactly two input fields, one of them named after the type it supports, and the other called 'render_config', of type 'dict' """ - _operation_type_name = "pretty_print" + _operation_type_name: ClassVar[str] = "pretty_print" def _calculate_op_id(self, source_type: str, target_type: str): diff --git a/src/kiara/operations/included_core_operations/render_data.py b/src/kiara/operations/included_core_operations/render_data.py index 2bbf85b01..cd6cf7de5 100644 --- a/src/kiara/operations/included_core_operations/render_data.py +++ b/src/kiara/operations/included_core_operations/render_data.py @@ -5,7 +5,7 @@ # # Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) -from typing import Dict, Iterable, Mapping, Union +from typing import ClassVar, Dict, Iterable, Mapping, Union import structlog from pydantic import Field @@ -33,7 +33,7 @@ class RenderDataOperationType(OperationType[RenderDataDetails]): """An operation that renders data (and metadata) associated with a value.""" - _operation_type_name = "render_data" + _operation_type_name: ClassVar[str] = "render_data" def _calculate_op_id(cls, source_type: str, target_type: str): diff --git a/src/kiara/operations/included_core_operations/render_value.py b/src/kiara/operations/included_core_operations/render_value.py index 7cf9c7d56..b0fe97a3f 100644 --- a/src/kiara/operations/included_core_operations/render_value.py +++ b/src/kiara/operations/included_core_operations/render_value.py @@ -5,7 +5,7 @@ # # Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) -from typing import Dict, Iterable, Mapping, Union +from typing import ClassVar, Dict, Iterable, Mapping, Union import structlog from pydantic import Field @@ -53,7 +53,7 @@ class RenderValueOperationType(OperationType[RenderValueDetails]): """ - _operation_type_name = "render_value" + _operation_type_name: ClassVar[str] = "render_value" def _calculate_op_id(cls, source_type: str, target_type: str): diff --git a/src/kiara/operations/included_core_operations/serialize.py b/src/kiara/operations/included_core_operations/serialize.py index e33d1dc9b..34fdd5010 100644 --- a/src/kiara/operations/included_core_operations/serialize.py +++ b/src/kiara/operations/included_core_operations/serialize.py @@ -5,7 +5,7 @@ # # Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Mapping, Union +from typing import TYPE_CHECKING, Any, ClassVar, Dict, Iterable, List, Mapping, Union from pydantic import Field @@ -78,7 +78,7 @@ class DeSerializeOperationType(OperationType[DeSerializeDetails]): - an input field called 'value' """ - _operation_type_name = "deserialize" + _operation_type_name: ClassVar[str] = "deserialize" def retrieve_included_operation_configs( self, diff --git a/src/kiara/registries/data/__init__.py b/src/kiara/registries/data/__init__.py index ffc9a2769..48c3e61f0 100644 --- a/src/kiara/registries/data/__init__.py +++ b/src/kiara/registries/data/__init__.py @@ -797,6 +797,9 @@ def _create_value( else: raise NotImplementedError() + if not pedigree.is_resolved: + pedigree = self._kiara.module_registry.resolve_manifest(pedigree) # type: ignore + v_id = ID_REGISTRY.generate( type="value", kiara_id=self._kiara.id, pre_registered=False ) diff --git a/src/kiara/registries/modules/__init__.py b/src/kiara/registries/modules/__init__.py index 3b2d63191..670af68d0 100644 --- a/src/kiara/registries/modules/__init__.py +++ b/src/kiara/registries/modules/__init__.py @@ -84,6 +84,35 @@ def get_context_metadata( return ModuleTypesInfo(group_title=alias, item_infos=result) # type: ignore + def resolve_manifest(self, manifest: Manifest) -> Manifest: + """Returns a cloned manifest with resolved module config.""" + + if manifest.is_resolved: + return manifest.model_copy() + + m_cls = self.get_module_class(manifest.module_type) + + try: + resolved = m_cls._resolve_module_config(**manifest.module_config) + resolved_dict = resolved.model_dump() + manifest_clone = manifest.model_copy( + update={"module_config": resolved_dict, "is_resolved": True} + ) + return manifest_clone + + except Exception as e: + if is_debug(): + import traceback + + traceback.print_exc() + + raise InvalidManifestException( + f"Error while resolving module config for module '{manifest.module_type}': {e}", + module_type=manifest.module_type, + module_config=manifest.module_config, + parent=e, + ) + def create_module(self, manifest: Union[Manifest, str]) -> "KiaraModule": """ Create a [KiaraModule][kiara.module.KiaraModule] object from a module configuration. @@ -98,23 +127,7 @@ def create_module(self, manifest: Union[Manifest, str]) -> "KiaraModule": m_cls: Type[KiaraModule] = self.get_module_class(manifest.module_type) if not manifest.is_resolved: - # dbg(manifest.module_config) - try: - resolved = m_cls._resolve_module_config(**manifest.module_config) - except Exception as e: - if is_debug(): - import traceback - - traceback.print_exc() - - raise InvalidManifestException( - f"Error while resolving module config for module '{manifest.module_type}': {e}", - module_type=manifest.module_type, - module_config=manifest.module_config, - parent=e, - ) - manifest.module_config = resolved.model_dump() - manifest.is_resolved = True + manifest = self.resolve_manifest(manifest) if self._cached_modules.setdefault(manifest.module_type, {}).get( manifest.instance_cid, None @@ -123,11 +136,7 @@ def create_module(self, manifest: Union[Manifest, str]) -> "KiaraModule": if manifest.module_type in self.get_module_type_names(): kiara_module = m_cls(module_config=manifest.module_config) - kiara_module._manifest_cache = Manifest( - module_type=manifest.module_type, - module_config=manifest.module_config, - is_resolved=manifest.is_resolved, - ) + kiara_module._manifest_cache = self.resolve_manifest(manifest) else: raise Exception( diff --git a/src/kiara/registries/operations/__init__.py b/src/kiara/registries/operations/__init__.py index bdaa0da7a..2c869d7dd 100644 --- a/src/kiara/registries/operations/__init__.py +++ b/src/kiara/registries/operations/__init__.py @@ -26,7 +26,10 @@ from rich.console import Group, RenderableType from ruamel.yaml import YAML -from kiara.exceptions import InvalidOperationException, NoSuchOperationException +from kiara.exceptions import ( + InvalidOperationException, + NoSuchOperationException, +) from kiara.interfaces.python_api.models.info import ( OperationTypeClassesInfo, OperationTypeInfo, @@ -207,7 +210,8 @@ def operations(self) -> Mapping[str, Operation]: for data_type in self._kiara.data_type_classes.values(): if hasattr(data_type, "retrieve_included_operations"): - for op in all_op_configs: + included_ops = data_type.retrieve_included_operations() + for op in included_ops: if isinstance(op, Mapping): op = ManifestOperationConfig(**op) all_op_configs.add(op) @@ -257,7 +261,6 @@ def operations(self) -> Mapping[str, Operation]: manifest = Manifest( module_type=module_type, module_config=module_config ) - ops = self._create_operations(manifest=manifest, doc=op_config.doc) for op_type_name, _op in ops.items(): diff --git a/src/kiara/utils/cli/run.py b/src/kiara/utils/cli/run.py index 2d38290f0..5ad601b9f 100644 --- a/src/kiara/utils/cli/run.py +++ b/src/kiara/utils/cli/run.py @@ -22,7 +22,7 @@ # from kiara.interfaces.python_api.operation import KiaraOperation from kiara.utils import log_exception -from kiara.utils.cli import dict_from_cli_args, terminal_print +from kiara.utils.cli import dict_from_cli_args, terminal_print, terminal_print_model from kiara.utils.cli.rich_click import rich_format_operation_help from kiara.utils.operations import create_operation_status_renderable from kiara.utils.output import create_table_from_base_model_cls @@ -332,6 +332,7 @@ def execute_job( silent: bool, save_results: bool, aliases: Union[None, Mapping[str, List[str]]], + properties: bool = False, ) -> uuid.UUID: """Execute the job.""" job_id = api.queue_job(operation=operation, inputs=inputs) @@ -367,6 +368,31 @@ def execute_job( outputs, in_panel=title, empty_line_before=True, show_data_type=True ) + if properties: + render_config = { + "show_pedigree": False, + "show_serialized": False, + "show_data_preview": False, + "show_properties": True, + "show_destinies": False, + "show_destiny_backlinks": False, + "show_lineage": False, + "show_environment_hashes": False, + "show_environment_data": False, + } + + title = "Result details" + format = "terminal" + + from kiara.interfaces.python_api import ValueInfo + + v_infos = ( + ValueInfo.create_from_instance(kiara=api.context, instance=v) + for v in outputs.values() + ) + + terminal_print_model(*v_infos, format=format, in_panel=title, **render_config) + # for k, v in outputs.items(): # rendered = kiara_obj.data_registry.render_data(v) # rich_print(rendered)