From 82f91cc3084e7e1261d0ee83f39df5db4b2e446f Mon Sep 17 00:00:00 2001 From: Markus Binsteiner Date: Sat, 2 Mar 2024 11:30:49 +0100 Subject: [PATCH] feat: add value created property to Value model This is a first implementation of a solution for #70, it is not considered stable yet, but please try it out and let me know if it does not work as expected. --- src/kiara/interfaces/cli/data/commands.py | 11 ++++++++ .../interfaces/python_api/models/info.py | 18 ++++++++++--- src/kiara/interfaces/python_api/workflow.py | 4 +-- src/kiara/models/values/value.py | 6 +++++ src/kiara/processing/__init__.py | 10 +++---- .../registries/data/data_store/__init__.py | 6 +++++ src/kiara/registries/workflows/__init__.py | 4 +-- src/kiara/utils/dates.py | 27 +++++++++++++++++++ 8 files changed, 74 insertions(+), 12 deletions(-) create mode 100644 src/kiara/utils/dates.py diff --git a/src/kiara/interfaces/cli/data/commands.py b/src/kiara/interfaces/cli/data/commands.py index 99ede1bcb..4a0e572a0 100644 --- a/src/kiara/interfaces/cli/data/commands.py +++ b/src/kiara/interfaces/cli/data/commands.py @@ -72,6 +72,13 @@ def data(ctx): default=False, is_flag=True, ) +@click.option( + "--date-created", + "-D", + help="Display the date when the value was created.", + default=False, + is_flag=True, +) @click.option( "--type-config", "-c", @@ -128,6 +135,7 @@ def list_values( hash, include_internal, value_id, + date_created, pedigree, data, type_config, @@ -164,6 +172,9 @@ def list_values( if not value_id and not all_values: render_fields.remove("value_id") + + if date_created: + render_fields.append("value_created") if type_config: render_fields.append("data_type_config") if hash: diff --git a/src/kiara/interfaces/python_api/models/info.py b/src/kiara/interfaces/python_api/models/info.py index 89a0d685c..65a084a58 100644 --- a/src/kiara/interfaces/python_api/models/info.py +++ b/src/kiara/interfaces/python_api/models/info.py @@ -6,6 +6,7 @@ import re import textwrap import uuid +from datetime import datetime from typing import ( TYPE_CHECKING, Any, @@ -68,6 +69,7 @@ from kiara.utils import log_exception, log_message from kiara.utils.class_loading import find_all_kiara_model_classes from kiara.utils.cli import HORIZONTALS_NO_TO_AND_BOTTOM +from kiara.utils.dates import to_human_readable_date_string from kiara.utils.json import orjson_dumps from kiara.utils.output import extract_renderable @@ -111,6 +113,12 @@ def pretty_print_value_data_terminal(value: "ValueInfo"): "render": {"terminal": lambda x: x.value_schema.type}, }, "value_schema": {"show_default": False}, + "value_created": { + "show_default": False, + "render": { + "terminal": lambda v: f"{to_human_readable_date_string(v.value_created)} ago" + }, + }, "is_persisted": { "show_default": False, "render": {"terminal": lambda v: "yes" if v.is_persisted else "no"}, @@ -497,6 +505,7 @@ def create_from_instance( value_id=instance.value_id, kiara_id=instance.kiara_id, value_schema=instance.value_schema, + value_created=instance.value_created, value_status=instance.value_status, environment_hashes=instance.environment_hashes, value_size=instance.value_size, @@ -529,7 +538,7 @@ def create_from_instance( value_schema: ValueSchema = Field( description="The schema that was used for this Value." ) - + value_created: datetime = Field(description="The time this value was created.") value_status: ValueStatus = Field(description="The set/unset status of this value.") value_size: int = Field(description="The size of this value, in bytes.") value_hash: str = Field(description="The hash of this value.") @@ -1939,18 +1948,21 @@ def create_renderable(self, **config: Any) -> RenderableType: class KiaraPluginInfos(InfoItemGroup[KiaraPluginInfo]): @classmethod def get_available_plugin_names( - cls, kiara: "Kiara", regex: str = "^kiara[-_]plugin\\..*" + cls, kiara: "Kiara", regex: Union[str, None] = None ) -> List[str]: """ Get a list of all available plugins. Arguments: - regex: an optional regex to indicate the plugin naming scheme (default: /$kiara[_-]plugin\..*/) + regex: an optional regex to indicate the plugin naming scheme (default: /$kiara[_-]plugin\\..*/) Returns: a list of plugin names """ + if regex is None: + regex = r"^kiara[-_]plugin\..*" + registry = kiara.environment_registry python_env: PythonRuntimeEnvironment = registry.environments["python"] # type: ignore diff --git a/src/kiara/interfaces/python_api/workflow.py b/src/kiara/interfaces/python_api/workflow.py index ef549b3ea..69355c0ee 100644 --- a/src/kiara/interfaces/python_api/workflow.py +++ b/src/kiara/interfaces/python_api/workflow.py @@ -3,7 +3,6 @@ from datetime import datetime from typing import TYPE_CHECKING, Any, Dict, Hashable, List, Mapping, Set, Tuple, Union -import pytz import structlog from boltons.strutils import slugify @@ -33,6 +32,7 @@ from kiara.models.workflow import WorkflowInfo, WorkflowMetadata, WorkflowState from kiara.registries.ids import ID_REGISTRY from kiara.utils import find_free_id, log_exception +from kiara.utils.dates import get_current_time_incl_timezone if TYPE_CHECKING: from kiara.context import Kiara @@ -1305,7 +1305,7 @@ def snapshot(self, save: bool = False) -> WorkflowState: if state.instance_id not in self._state_cache.keys(): self._state_cache[state.instance_id] = state - now = datetime.now(pytz.utc) + now = get_current_time_incl_timezone() for field_name, value in self.current_pipeline_outputs.items(): if value in [NOT_SET_VALUE_ID, NONE_VALUE_ID]: diff --git a/src/kiara/models/values/value.py b/src/kiara/models/values/value.py index 87ca34866..53bac471b 100644 --- a/src/kiara/models/values/value.py +++ b/src/kiara/models/values/value.py @@ -12,6 +12,7 @@ import os import tempfile import uuid +from datetime import datetime from typing import ( TYPE_CHECKING, Any, @@ -55,6 +56,7 @@ from kiara.models.values import DataTypeCharacteristics, ValueStatus from kiara.models.values.value_schema import ValueSchema from kiara.utils import is_jupyter, log_exception +from kiara.utils.dates import get_current_time_incl_timezone from kiara.utils.hashing import create_cid_digest from kiara.utils.json import orjson_dumps from kiara.utils.yaml import StringYAML @@ -682,6 +684,10 @@ class ValueDetails(KiaraModel): description="The schema that was used for this Value." ) + value_created: datetime = Field( + description="The time when this value was created.", + default_factory=get_current_time_incl_timezone, + ) value_status: ValueStatus = Field(description="The set/unset status of this value.") value_size: int = Field(description="The size of this value, in bytes.") value_hash: str = Field(description="The hash of this value.") diff --git a/src/kiara/processing/__init__.py b/src/kiara/processing/__init__.py index 4f43873b1..5e605070a 100644 --- a/src/kiara/processing/__init__.py +++ b/src/kiara/processing/__init__.py @@ -7,7 +7,6 @@ import abc import uuid -from datetime import datetime from typing import TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Protocol, Union import structlog @@ -24,6 +23,7 @@ from kiara.modules import KiaraModule from kiara.registries.ids import ID_REGISTRY from kiara.utils import get_dev_config, is_develop, log_exception +from kiara.utils.dates import get_current_time_incl_timezone if TYPE_CHECKING: from kiara.context import Kiara @@ -256,7 +256,7 @@ def job_status_updated( self._active_jobs.pop(job_id) job.job_log.add_log("job finished successfully") job.status = JobStatus.SUCCESS - job.finished = datetime.now() + job.finished = get_current_time_incl_timezone() values = self._output_refs[job_id] try: values.sync_values() @@ -272,7 +272,7 @@ def job_status_updated( status = e job.job_log.add_log("job failed") job.status = JobStatus.FAILED - job.finished = datetime.now() + job.finished = get_current_time_incl_timezone() msg = str(status) job.error = msg job._exception = status @@ -290,7 +290,7 @@ def job_status_updated( self._active_jobs.pop(job_id) job.job_log.add_log("job failed") job.status = JobStatus.FAILED - job.finished = datetime.now() + job.finished = get_current_time_incl_timezone() if isinstance(status, str): job.error = status elif isinstance(status, Exception): @@ -308,7 +308,7 @@ def job_status_updated( elif status == JobStatus.STARTED: job.job_log.add_log("job started") job.status = JobStatus.STARTED - job.started = datetime.now() + job.started = get_current_time_incl_timezone() else: raise ValueError(f"Invalid value for status: {status}") diff --git a/src/kiara/registries/data/data_store/__init__.py b/src/kiara/registries/data/data_store/__init__.py index 275ab6148..b19d09cf4 100644 --- a/src/kiara/registries/data/data_store/__init__.py +++ b/src/kiara/registries/data/data_store/__init__.py @@ -36,6 +36,7 @@ ) from kiara.models.values.value_schema import ValueSchema from kiara.registries import ARCHIVE_CONFIG_CLS, BaseArchive +from kiara.utils.dates import get_earliest_time_incl_timezone if TYPE_CHECKING: from multiformats import CID @@ -123,11 +124,16 @@ def retrieve_value(self, value_id: uuid.UUID) -> Value: # data_type=value_schema.type, data_type_config=value_schema.type_config # ) + value_created = value_data.get("value_created", None) + if value_created is None: + value_created = get_earliest_time_incl_timezone() + pedigree = ValuePedigree(**value_data["pedigree"]) value = Value( value_id=value_data["value_id"], kiara_id=self.kiara_context.id, value_schema=value_schema, + value_created=value_created, value_status=value_data["value_status"], value_size=value_data["value_size"], value_hash=value_data["value_hash"], diff --git a/src/kiara/registries/workflows/__init__.py b/src/kiara/registries/workflows/__init__.py index 98c325337..4dfc61874 100644 --- a/src/kiara/registries/workflows/__init__.py +++ b/src/kiara/registries/workflows/__init__.py @@ -18,7 +18,6 @@ Union, ) -import pytz import structlog from kiara.exceptions import NoSuchWorkflowException @@ -26,6 +25,7 @@ from kiara.models.workflow import WorkflowMetadata, WorkflowState from kiara.registries import ARCHIVE_CONFIG_CLS, BaseArchive from kiara.registries.ids import ID_REGISTRY +from kiara.utils.dates import get_current_time_incl_timezone if TYPE_CHECKING: from kiara.context import Kiara @@ -464,7 +464,7 @@ def add_workflow_state( workflow_details = self.get_workflow_metadata(workflow=workflow) if timestamp is None: - timestamp = datetime.datetime.now(pytz.utc) + timestamp = get_current_time_incl_timezone() if timestamp in workflow_details.workflow_history.keys(): if ( diff --git a/src/kiara/utils/dates.py b/src/kiara/utils/dates.py new file mode 100644 index 000000000..dc0976d37 --- /dev/null +++ b/src/kiara/utils/dates.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +import time +from datetime import datetime + +import humanfriendly +import pytz + + +def get_current_time_incl_timezone() -> datetime: + + current_tz_name = time.tzname[0] + current_tz = pytz.timezone(current_tz_name) + + return datetime.now(tz=current_tz) + + +def get_earliest_time_incl_timezone() -> datetime: + return datetime(1970, 1, 1, tzinfo=pytz.utc) + + +def to_human_readable_date_string(datetime: datetime) -> str: + + now = get_current_time_incl_timezone() + time_gone = (now - datetime).total_seconds() + + relative_time_str: str = humanfriendly.format_timespan(time_gone, max_units=1) + return relative_time_str