Feature/multi-user

pipelex/builder/builder_validation.py

@@ -21,6 +21,7 @@
     StaticValidationError,
 )
 from pipelex.hub import get_library_manager
+from pipelex.libraries.library_ids import SpecialLibraryId
 from pipelex.pipe_run.dry_run import DryRunOutput, dry_run_pipes
 
 
@@ -41,7 +42,7 @@ async def validate_bundle_spec(bundle_spec: PipelexBundleSpec):
 
     library_manager = get_library_manager()
     dry_run_result = await dry_run_bundle_blueprint(bundle_blueprint=bundle_blueprint)
-    library_manager.remove_from_blueprint(blueprint=bundle_blueprint)
+    library_manager.remove_from_blueprints(library_id=SpecialLibraryId.UNTITLED, blueprints=[bundle_blueprint])
 
     dry_run_pipe_failures = extract_pipe_failures_from_dry_run_result(bundle_spec=bundle_spec, dry_run_result=dry_run_result)
     if dry_run_pipe_failures:
@@ -103,7 +104,7 @@ def document_pipe_failures_from_dry_run_blueprint(
 async def dry_run_bundle_blueprint(bundle_blueprint: PipelexBundleBlueprint) -> dict[str, DryRunOutput]:
     library_manager = get_library_manager()
     try:
-        pipes = library_manager.load_from_blueprint(blueprint=bundle_blueprint)
+        pipes = library_manager.load_from_blueprints(library_id=SpecialLibraryId.UNTITLED, blueprints=[bundle_blueprint])
         dry_run_result = await dry_run_pipes(pipes=pipes, raise_on_failure=True)
     except StaticValidationError as static_validation_error:
         static_validation_error_data = StaticValidationErrorData(

pipelex/cli/commands/validate_cmd.py

@@ -13,16 +13,14 @@
 
 def do_validate_all_libraries_and_dry_run() -> None:
     """Validate libraries and dry-run all pipes."""
-    pipelex_instance = Pipelex.make()
-    pipelex_instance.validate_libraries()
+    Pipelex.make()
     asyncio.run(dry_run_pipes(pipes=get_pipes(), raise_on_failure=True))
     log.info("Setup sequence passed OK, config and pipelines are validated.")
 
 
 def do_dry_run_pipe(pipe_code: str) -> None:
     """Dry run a single pipe."""
-    pipelex_instance = Pipelex.make()
-    pipelex_instance.validate_libraries()
+    Pipelex.make()
 
     asyncio.run(
         dry_run_pipe(

pipelex/core/stuffs/stuff_factory.py

@@ -7,7 +7,6 @@
 from pipelex.core.concepts.concept import Concept
 from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
 from pipelex.core.concepts.concept_factory import ConceptFactory
-from pipelex.core.concepts.concept_library import ConceptLibraryConceptNotFoundError
 from pipelex.core.concepts.concept_native import NativeConceptCode
 from pipelex.core.stuffs.list_content import ListContent
 from pipelex.core.stuffs.structured_content import StructuredContent
@@ -16,6 +15,7 @@
 from pipelex.core.stuffs.text_content import TextContent
 from pipelex.exceptions import PipelexException
 from pipelex.hub import get_class_registry, get_concept_library, get_native_concept, get_required_concept
+from pipelex.libraries.concept.concept_library import ConceptLibraryConceptNotFoundError
 from pipelex.tools.typing.pydantic_utils import format_pydantic_validation_error
 
 

pipelex/hub.py

@@ -13,17 +13,17 @@
 from pipelex.cogt.models.model_deck import ModelDeck
 from pipelex.cogt.models.model_manager_abstract import ModelManagerAbstract
 from pipelex.core.concepts.concept import Concept
-from pipelex.core.concepts.concept_library_abstract import ConceptLibraryAbstract
 from pipelex.core.concepts.concept_native import NativeConceptCode
 from pipelex.core.domains.domain import Domain
-from pipelex.core.domains.domain_library_abstract import DomainLibraryAbstract
 from pipelex.core.pipes.pipe_abstract import PipeAbstract
-from pipelex.core.pipes.pipe_library_abstract import PipeLibraryAbstract
+from pipelex.libraries.concept.concept_library_abstract import ConceptLibraryAbstract
+from pipelex.libraries.domain.domain_library_abstract import DomainLibraryAbstract
+from pipelex.libraries.library_ids import SpecialLibraryId
 from pipelex.libraries.library_manager_abstract import LibraryManagerAbstract
+from pipelex.libraries.pipe.pipe_library_abstract import PipeLibraryAbstract
 from pipelex.observer.observer_protocol import ObserverProtocol
 from pipelex.pipe_run.pipe_router_protocol import PipeRouterProtocol
 from pipelex.pipeline.activity.activity_manager_protocol import ActivityManagerProtocol
-from pipelex.pipeline.pipeline import Pipeline
 from pipelex.pipeline.pipeline_manager_abstract import PipelineManagerAbstract
 from pipelex.pipeline.track.pipeline_tracker_protocol import PipelineTrackerProtocol
 from pipelex.plugins.plugin_manager import PluginManager
@@ -233,21 +233,27 @@ def get_required_content_generator(self) -> ContentGeneratorProtocol:
 
     # pipelex
 
-    def get_required_domain_library(self) -> DomainLibraryAbstract:
+    def get_required_domain_library(self, library_id: str | None = None) -> DomainLibraryAbstract:
+        if self._library_manager is not None:
+            return self._library_manager.get_library(library_id=library_id).domain_library
         if self._domain_library is None:
-            msg = "DomainLibrary is not initialized"
+            msg = "DomainLibrary with library_id '{library_id}' is not initialized"
             raise RuntimeError(msg)
         return self._domain_library
 
-    def get_required_concept_library(self) -> ConceptLibraryAbstract:
+    def get_required_concept_library(self, library_id: str | None = None) -> ConceptLibraryAbstract:
+        if self._library_manager is not None:
+            return self._library_manager.get_library(library_id=library_id).concept_library
         if self._concept_library is None:
-            msg = "ConceptLibrary is not initialized"
+            msg = "ConceptLibrary    with library_id '{library_id}' is not initialized"
             raise RuntimeError(msg)
         return self._concept_library
 
-    def get_required_pipe_library(self) -> PipeLibraryAbstract:
+    def get_required_pipe_library(self, library_id: str | None = None) -> PipeLibraryAbstract:
+        if self._library_manager is not None:
+            return self._library_manager.get_library(library_id=library_id).pipe_library
         if self._pipe_library is None:
-            msg = "PipeLibrary is not initialized"
+            msg = "PipeLibrary with library_id '{library_id}' is not initialized"
             raise RuntimeError(msg)
         return self._pipe_library
 
@@ -372,36 +378,36 @@ def get_secret(secret_id: str) -> str:
     return get_secrets_provider().get_secret(secret_id=secret_id)
 
 
-def get_required_domain(domain: str) -> Domain:
-    return get_pipelex_hub().get_required_domain_library().get_required_domain(domain=domain)
+def get_required_domain(domain: str, library_id: str | None = None) -> Domain:
+    return get_pipelex_hub().get_required_domain_library(library_id=library_id).get_required_domain(domain=domain)
 
 
-def get_optional_domain(domain: str) -> Domain | None:
-    return get_pipelex_hub().get_required_domain_library().get_domain(domain=domain)
+def get_optional_domain(domain: str, library_id: str | None = None) -> Domain | None:
+    return get_pipelex_hub().get_required_domain_library(library_id=library_id).get_domain(domain=domain)
 
 
-def get_pipe_library() -> PipeLibraryAbstract:
-    return get_pipelex_hub().get_required_pipe_library()
+def get_pipe_library(library_id: str | None = None) -> PipeLibraryAbstract:
+    return get_pipelex_hub().get_required_pipe_library(library_id=library_id)
 
 
-def get_pipes() -> list[PipeAbstract]:
-    return get_pipelex_hub().get_required_pipe_library().get_pipes()
+def get_pipes(library_id: str | None = None) -> list[PipeAbstract]:
+    return get_pipelex_hub().get_required_pipe_library(library_id=library_id).get_pipes()
 
 
-def get_required_pipe(pipe_code: str) -> PipeAbstract:
-    return get_pipelex_hub().get_required_pipe_library().get_required_pipe(pipe_code=pipe_code)
+def get_required_pipe(pipe_code: str, library_id: str | None = None) -> PipeAbstract:
+    return get_pipelex_hub().get_required_pipe_library(library_id=library_id).get_required_pipe(pipe_code=pipe_code)
 
 
-def get_optional_pipe(pipe_code: str) -> PipeAbstract | None:
-    return get_pipelex_hub().get_required_pipe_library().get_optional_pipe(pipe_code=pipe_code)
+def get_optional_pipe(pipe_code: str, library_id: str | None = None) -> PipeAbstract | None:
+    return get_pipelex_hub().get_required_pipe_library(library_id=library_id).get_optional_pipe(pipe_code=pipe_code)
 
 
-def get_concept_library() -> ConceptLibraryAbstract:
-    return get_pipelex_hub().get_required_concept_library()
+def get_concept_library(library_id: str | None = None) -> ConceptLibraryAbstract:
+    return get_pipelex_hub().get_required_concept_library(library_id=library_id)
 
 
-def get_required_concept(concept_string: str) -> Concept:
-    return get_pipelex_hub().get_required_concept_library().get_required_concept(concept_string=concept_string)
+def get_required_concept(concept_string: str, library_id: str | None = None) -> Concept:
+    return get_pipelex_hub().get_required_concept_library(library_id=library_id).get_required_concept(concept_string=concept_string)
 
 
 def get_pipe_router() -> PipeRouterProtocol:
@@ -420,10 +426,6 @@ def get_activity_manager() -> ActivityManagerProtocol:
     return get_pipelex_hub().get_activity_manager()
 
 
-def get_pipeline(pipeline_run_id: str) -> Pipeline:
-    return get_pipeline_manager().get_pipeline(pipeline_run_id=pipeline_run_id)
-
-
 def get_library_manager() -> LibraryManagerAbstract:
     return get_pipelex_hub().get_required_library_manager()
 
@@ -433,4 +435,4 @@ def get_observer_provider() -> ObserverProtocol:
 
 
 def get_native_concept(native_concept: NativeConceptCode) -> Concept:
-    return get_pipelex_hub().get_required_concept_library().get_native_concept(native_concept=native_concept)
+    return get_pipelex_hub().get_required_concept_library(library_id=SpecialLibraryId.UNTITLED).get_native_concept(native_concept=native_concept)

pipelex/libraries/__init__.py

@@ -1 +1,11 @@
+from pipelex.libraries.library import Library
+from pipelex.libraries.library_ids import SpecialLibraryId
+from pipelex.libraries.library_manager import LibraryManager
+from pipelex.libraries.library_manager_abstract import LibraryManagerAbstract
 
+__all__ = [
+    "Library",
+    "LibraryManager",
+    "LibraryManagerAbstract",
+    "SpecialLibraryId",
+]

pipelex/core/concepts/concept_library.py → pipelex/libraries/concept/concept_library.py

@@ -1,17 +1,13 @@
-from typing import Any
-
-from pydantic import Field, RootModel
+from pydantic import Field, RootModel, model_validator
 from typing_extensions import override
 
 from pipelex.core.concepts.concept import Concept
 from pipelex.core.concepts.concept_blueprint import ConceptBlueprint
 from pipelex.core.concepts.concept_factory import ConceptFactory
-from pipelex.core.concepts.concept_library_abstract import ConceptLibraryAbstract
 from pipelex.core.concepts.concept_native import NativeConceptCode
 from pipelex.core.domains.domain import SpecialDomain
-from pipelex.core.stuffs.image_content import ImageContent
 from pipelex.exceptions import ConceptLibraryConceptNotFoundError, ConceptLibraryError
-from pipelex.hub import get_class_registry
+from pipelex.libraries.concept.concept_library_abstract import ConceptLibraryAbstract
 from pipelex.types import Self
 
 ConceptLibraryRoot = dict[str, Concept]
@@ -20,8 +16,8 @@
 class ConceptLibrary(RootModel[ConceptLibraryRoot], ConceptLibraryAbstract):
     root: ConceptLibraryRoot = Field(default_factory=dict)
 
-    def validate_with_libraries(self):
-        """Validates that the each refine concept code in the refines array of each concept in the library exists in the library"""
+    @model_validator(mode="before")
+    def validation_static(self):
         for concept in self.root.values():
             if concept.refines and concept.refines not in self.root:
                 msg = f"Concept '{concept.code}' refines '{concept.refines}' but no concept with the code '{concept.refines}' exists"
@@ -33,17 +29,19 @@ def setup(self):
         self.add_concepts(concepts=all_native_concepts)
 
     @override
-    def reset(self):
+    def teardown(self):
         self.root = {}
-        self.setup()
 
     @override
-    def teardown(self):
-        self.root = {}
+    def reset(self):
+        self.teardown()
+        self.setup()
 
     @classmethod
     def make_empty(cls) -> Self:
-        return cls(root={})
+        library = cls(root={})
+        library.setup()
+        return library
 
     @override
     def list_concepts(self) -> list[Concept]:
@@ -105,23 +103,15 @@ def get_native_concepts(self) -> list[Concept]:
         return [self.get_native_concept(native_concept=native_concept) for native_concept in NativeConceptCode.values_list()]
 
     @override
-    def get_class(self, concept_code: str) -> type[Any] | None:
-        return get_class_registry().get_class(concept_code)
+    def search_for_concept_in_domains(self, concept_code: str, search_domains: list[str]) -> Concept | None:
+        ConceptBlueprint.validate_concept_code(concept_code=concept_code)
+        for domain in search_domains:
+            if found_concept := self.get_required_concept(
+                concept_string=ConceptFactory.make_concept_string_with_domain(domain=domain, concept_code=concept_code),
+            ):
+                return found_concept
 
-    @override
-    def is_image_concept(self, concept: Concept) -> bool:
-        """Check if the concept is an image concept.
-        It is an image concept if its structure class is a subclass of ImageContent
-        or if it refines the native Image concept.
-        """
-        pydantic_model = self.get_class(concept_code=concept.structure_class_name)
-        is_image_class = bool(pydantic_model and issubclass(pydantic_model, ImageContent))
-        refines_image = self.is_compatible(
-            tested_concept=concept,
-            wanted_concept=self.get_native_concept(native_concept=NativeConceptCode.IMAGE),
-            strict=True,
-        )
-        return is_image_class or refines_image
+        return None
 
     @override
     def get_required_concept_from_concept_string_or_code(self, concept_string_or_code: str, search_domains: list[str] | None = None) -> Concept:
@@ -155,14 +145,3 @@ def get_required_concept_from_concept_string_or_code(self, concept_string_or_cod
                     msg = f"Multiple concepts found for '{concept_string_or_code}': {found_concepts}. Please specify the domain."
                     raise ConceptLibraryConceptNotFoundError(msg)
                 return found_concepts[0]
-
-    @override
-    def search_for_concept_in_domains(self, concept_code: str, search_domains: list[str]) -> Concept | None:
-        ConceptBlueprint.validate_concept_code(concept_code=concept_code)
-        for domain in search_domains:
-            if found_concept := self.get_required_concept(
-                concept_string=ConceptFactory.make_concept_string_with_domain(domain=domain, concept_code=concept_code),
-            ):
-                return found_concept
-
-        return None

pipelex/core/concepts/concept_library_abstract.py → pipelex/libraries/concept/concept_library_abstract.py

@@ -1,5 +1,4 @@
 from abc import ABC, abstractmethod
-from typing import Any
 
 from pipelex.core.concepts.concept import Concept
 from pipelex.core.concepts.concept_native import NativeConceptCode
@@ -46,18 +45,10 @@ def reset(self) -> None:
     def teardown(self) -> None:
         pass
 
-    @abstractmethod
-    def is_image_concept(self, concept: Concept) -> bool:
-        pass
-
     @abstractmethod
     def search_for_concept_in_domains(self, concept_code: str, search_domains: list[str]) -> Concept | None:
         pass
 
-    @abstractmethod
-    def get_class(self, concept_code: str) -> type[Any] | None:
-        pass
-
     @abstractmethod
     def get_native_concept(self, native_concept: NativeConceptCode) -> Concept:
         pass

pipelex/core/domains/domain_library.py → pipelex/libraries/domain/domain_library.py

@@ -2,8 +2,8 @@
 from typing_extensions import override
 
 from pipelex.core.domains.domain import Domain
-from pipelex.core.domains.domain_library_abstract import DomainLibraryAbstract
 from pipelex.exceptions import DomainLibraryError
+from pipelex.libraries.domain.domain_library_abstract import DomainLibraryAbstract
 from pipelex.types import Self
 
 DomainLibraryRoot = dict[str, Domain]
@@ -13,9 +13,17 @@ class DomainLibrary(RootModel[DomainLibraryRoot], DomainLibraryAbstract):
     def validate_with_libraries(self):
         pass
 
-    def reset(self):
+    def setup(self):
+        pass
+
+    @override
+    def teardown(self):
         self.root = {}
 
+    def reset(self):
+        self.teardown()
+        self.setup()
+
     @classmethod
     def make_empty(cls) -> Self:
         return cls(root={})
@@ -36,18 +44,11 @@ def remove_domain_by_code(self, domain_code: str) -> None:
         if domain_code in self.root:
             del self.root[domain_code]
 
-    @override
-    def get_domain(self, domain: str) -> Domain | None:
-        return self.root.get(domain)
-
     @override
     def get_required_domain(self, domain: str) -> Domain:
+        """Get a domain by code from this library, raising an error if not found."""
         the_domain = self.get_domain(domain=domain)
         if not the_domain:
             msg = f"Domain '{domain}' not found. Check for typos and make sure it is declared in a pipeline library."
             raise DomainLibraryError(msg)
         return the_domain
-
-    @override
-    def teardown(self) -> None:
-        self.root = {}

pipelex/core/domains/domain_library_abstract.py → pipelex/libraries/domain/domain_library_abstract.py

@@ -6,11 +6,11 @@
 class DomainLibraryAbstract(ABC):
     @abstractmethod
     def get_domain(self, domain: str) -> Domain | None:
-        pass
+        """Get a domain by code from this library."""
 
     @abstractmethod
     def get_required_domain(self, domain: str) -> Domain:
-        pass
+        """Get a domain by code from this library, raising an error if not found."""
 
     @abstractmethod
     def teardown(self) -> None: