Merge pull request #190 from ORNL/dev

Dev

.github/workflows/create-release-n-publish.yml

@@ -8,6 +8,9 @@ jobs:
   build:
     name: Create Release and Publish
     runs-on: ubuntu-latest
+    env:
+      MONGO_ENABLED: false
+      LMDB_ENABLED: true
     timeout-minutes: 60
     steps:
       - name: Checkout code

.github/workflows/run-tests-py11-all-dbs.yml → .github/workflows/run-tests-all-dbs.yml

@@ -1,11 +1,14 @@
-name: (With and Without Mongo) Tests on py11
+name: With and Without Mongo at the same time
 on: [pull_request]
 
 jobs:
 
   build:
     runs-on: ubuntu-latest
     timeout-minutes: 40
+    strategy:
+      matrix:
+        python-version: [ "3.10", "3.11", "3.12" ]
     env:
       MONGO_ENABLED: true
       LMDB_ENABLED: true
@@ -17,10 +20,10 @@ jobs:
         with:
           fetch-depth: 1
 
-      - name: Set up Python 3.11
+      - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: ${{ matrix.python-version }}
           cache: "pip"
 
       - name: Show OS Info

.github/workflows/run-tests-in-container.yml

@@ -1,4 +1,4 @@
-name: (With and Without Mongo) Tests inside a Container Tests inside a Container
+name: (With and Without Mongo) Inside a Container
 on: [pull_request]
 
 jobs:

.github/workflows/run-tests-kafka.yml

@@ -10,6 +10,9 @@ jobs:
 
   build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ "3.10", "3.11", "3.12" ]
     env:
       MONGO_ENABLED: true
       LMDB_ENABLED: false
@@ -21,10 +24,10 @@ jobs:
         with:
           fetch-depth: 1
 
-      - name: Set up Python 3.10
+      - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: ${{ matrix.python-version }}
           cache: "pip"
 
       - name: Run docker compose

.github/workflows/run-tests-simple.yml

@@ -8,6 +8,9 @@ jobs:
 
   build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ "3.10", "3.11", "3.12" ]
     env:
       MONGO_ENABLED: false
       LMDB_ENABLED: true
@@ -19,10 +22,10 @@ jobs:
         with:
           fetch-depth: 1
 
-      - name: Set up Python 3.10
+      - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: ${{ matrix.python-version }}
           cache: "pip"
 
       - name: Show OS Info

.github/workflows/run-tests.yml

@@ -1,4 +1,4 @@
-name: (With Mongo) Unit, integration, and notebook tests
+name: Main Tests
 on:
   push:
   schedule:
@@ -8,6 +8,9 @@ jobs:
 
   build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ "3.10", "3.11", "3.12" ]
     env:
       MONGO_ENABLED: true
       LMDB_ENABLED: false
@@ -19,10 +22,10 @@ jobs:
         with:
           fetch-depth: 1
 
-      - name: Set up Python 3.10
+      - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: ${{ matrix.python-version }}
           cache: "pip"
 
       - name: Show OS Info

Makefile

@@ -25,7 +25,7 @@ checks:
 	ruff format --check src
 
 reformat:
-	ruff check src --fix
+	ruff check src --fix --unsafe-fixes
 	ruff format src
 
 # Remove cache directories and Sphinx build output

pyproject.toml

@@ -101,7 +101,7 @@ line-length = 100
 
 [tool.ruff.lint]
 extend-select = ["E501", "D"]
-ignore = ["D200", "D212", "D105", "D401"]
+ignore = ["D200", "D212", "D105", "D401", "D205", "D100"]
 
 [tool.ruff.lint.pydocstyle]
 convention = "numpy"

src/flowcept/__init__.py

@@ -1,6 +1,5 @@
 """Flowcept package."""
 
-from flowcept.configs import SETTINGS_PATH
 from flowcept.version import __version__
 
 from flowcept.commons.flowcept_dataclasses.workflow_object import (
@@ -19,6 +18,11 @@ def __getattr__(name):
 
         return flowcept_task
 
+    elif name == "FlowceptTask":
+        from flowcept.instrumentation.task_capture import FlowceptTask
+
+        return FlowceptTask
+
     elif name == "flowcept_torch":
         from flowcept.instrumentation.flowcept_torch import flowcept_torch
 
@@ -34,6 +38,11 @@ def __getattr__(name):
 
         return telemetry_flowcept_task
 
+    elif name == "lightweight_flowcept_task":
+        from flowcept.instrumentation.flowcept_task import lightweight_flowcept_task
+
+        return lightweight_flowcept_task
+
     if name == "MLFlowInterceptor":
         from flowcept.flowceptor.adapters.mlflow.mlflow_interceptor import (
             MLFlowInterceptor,
@@ -58,12 +67,10 @@ def __getattr__(name):
         )
 
         return TensorboardInterceptor
-    elif name == "ZambezeInterceptor":
-        from flowcept.flowceptor.adapters.zambeze.zambeze_interceptor import (
-            ZambezeInterceptor,
-        )
+    elif name == "SETTINGS_PATH":
+        from configs import SETTINGS_PATH
 
-        return ZambezeInterceptor
+        return SETTINGS_PATH
     elif name == "TaskQueryAPI":
         from flowcept.flowcept_api.task_query_api import TaskQueryAPI
 
@@ -80,7 +87,9 @@ def __getattr__(name):
     "TaskQueryAPI",
     "flowcept_task",
     "FlowceptLoop",
+    "FlowceptTask",
     "telemetry_flowcept_task",
+    "lightweight_flowcept_task",
     "Flowcept",
     "flowcept_torch",
     "WorkflowObject",

src/flowcept/commons/daos/docdb_dao/lmdb_dao.py

@@ -12,7 +12,7 @@
 
 from flowcept import WorkflowObject
 from flowcept.commons.daos.docdb_dao.docdb_dao_base import DocumentDBDAO
-from flowcept.configs import PERF_LOG, DATABASES
+from flowcept.configs import PERF_LOG, LMDB_SETTINGS
 from flowcept.flowceptor.consumers.consumer_utils import curate_dict_task_messages
 
 
@@ -36,7 +36,7 @@ def __init__(self):
 
     def _open(self):
         """Open LMDB environment and databases."""
-        _path = DATABASES.get("lmdb").get("path", "lmdb")
+        _path = LMDB_SETTINGS.get("path", "flowcept_lmdb")
         self._env = lmdb.open(_path, map_size=10**12, max_dbs=2)
         self._tasks_db = self._env.open_db(b"tasks")
         self._workflows_db = self._env.open_db(b"workflows")

src/flowcept/commons/flowcept_dataclasses/workflow_object.py

@@ -2,7 +2,7 @@
 
 from typing import Dict, AnyStr, List
 import msgpack
-from omegaconf import OmegaConf
+from omegaconf import OmegaConf, DictConfig
 
 from flowcept.version import __version__
 from flowcept.commons.utils import get_utc_now
@@ -70,7 +70,9 @@ def to_dict(self):
     def enrich(self, adapter_key=None):
         """Enrich it."""
         self.utc_timestamp = get_utc_now()
-        self.flowcept_settings = OmegaConf.to_container(settings)
+        self.flowcept_settings = (
+            OmegaConf.to_container(settings) if isinstance(settings, DictConfig) else settings
+        )
 
         if adapter_key is not None:
             # TODO :base-interceptor-refactor: :code-reorg: :usability:
@@ -90,7 +92,12 @@ def enrich(self, adapter_key=None):
             self.sys_name = SYS_NAME
 
         if self.extra_metadata is None and EXTRA_METADATA is not None:
-            self.extra_metadata = OmegaConf.to_container(EXTRA_METADATA)
+            _extra_metadata = (
+                OmegaConf.to_container(EXTRA_METADATA)
+                if isinstance(EXTRA_METADATA, DictConfig)
+                else EXTRA_METADATA
+            )
+            self.extra_metadata = _extra_metadata
 
         if self.flowcept_version is None:
             self.flowcept_version = __version__