diff --git a/.codeboarding/CLI_User_Interface.json b/.codeboarding/CLI_User_Interface.json new file mode 100644 index 000000000..6e49c04a9 --- /dev/null +++ b/.codeboarding/CLI_User_Interface.json @@ -0,0 +1,155 @@ +{ + "description": "The Pipelex system's command-line interface is orchestrated by the PipelexCLI component, which acts as the central dispatcher for user commands. It delegates specific tasks to CLI Commands, such as run, build, and validate. These commands, in turn, interact with the Pipelex Core, which embodies the fundamental logic for pipeline execution, validation, and bundle management. The Pipelex Core relies on Pipelex System Configuration for managing settings and data, and directly manages Pipelex Pipeline instances. The Pipelex Builder component is responsible for generating pipelines based on user requirements, a function initiated by relevant CLI Commands. This architecture ensures a clear separation of concerns, with the CLI handling user interaction and command dispatch, while the core components manage the underlying pipeline logic and data.", + "components": [ + { + "name": "PipelexCLI", + "description": "Serves as the primary entry point for the command-line interface. It initializes the CLI application, registers available commands, parses user input, and dispatches control to the appropriate command handler. It acts as the orchestrator for user interactions with the Pipelex system.", + "referenced_source_code": [ + { + "qualified_name": "pipelex.cli._cli.PipelexCLI", + "reference_file": "pipelex/cli/_cli.py", + "reference_start_line": 14, + "reference_end_line": 27 + } + ], + "assigned_files": [ + "pipelex/cli/_cli.py", + "pipelex/cli/__init__.py" + ], + "can_expand": true + }, + { + "name": "CLI Commands", + "description": "Each module within `pipelex.cli.commands` implements the specific business logic for a particular CLI command (e.g., run, build, validate). This includes handling command-line arguments, validating inputs, and invoking the relevant core Pipelex functionalities to perform tasks such as running, building, validating, or managing pipelines and concepts.", + "referenced_source_code": [ + { + "qualified_name": "pipelex.cli.commands.run_cmd.run_cmd", + "reference_file": "pipelex/cli/commands/run_cmd.py", + "reference_start_line": 19, + "reference_end_line": 182 + }, + { + "qualified_name": "pipelex.cli.commands.build_cmd.build_app", + "reference_file": "pipelex/cli/commands/build_cmd.py", + "reference_start_line": 24, + "reference_end_line": 24 + }, + { + "qualified_name": "pipelex.cli.commands.validate_cmd.validate_cmd", + "reference_file": "pipelex/cli/commands/validate_cmd.py", + "reference_start_line": 33, + "reference_end_line": 148 + } + ], + "assigned_files": [ + "pipelex/cli/commands/__init__.py", + "pipelex/cli/commands/build_cmd.py", + "pipelex/cli/commands/kit_cmd.py", + "pipelex/cli/commands/init_cmd.py", + "pipelex/cli/commands/validate_cmd.py", + "pipelex/cli/commands/run_cmd.py", + "pipelex/cli/commands/show_cmd.py" + ], + "can_expand": true + }, + { + "name": "Pipelex Core", + "description": "Encapsulates the central functionalities of the Pipelex system, including pipeline execution, validation logic, and interaction with bundles. It provides the underlying mechanisms for processing and managing pipelines and their associated data.", + "referenced_source_code": [ + { + "qualified_name": "pipelex.core.Pipelex", + "reference_file": "pipelex/pipelex.py", + "reference_start_line": 64, + "reference_end_line": 292 + }, + { + "qualified_name": "pipelex.core.interpreter.PipelexInterpreter", + "reference_file": "pipelex/core/interpreter.py", + "reference_start_line": 17, + "reference_end_line": 92 + } + ], + "assigned_files": [], + "can_expand": true + }, + { + "name": "Pipelex System Configuration", + "description": "Manages the configuration settings and input/output data for the Pipelex system. This includes loading and saving JSON files for inputs and outputs, and potentially other system-wide settings.", + "referenced_source_code": [ + { + "qualified_name": "pipelex.utils.io.load_json_dict_from_path", + "reference_file": "pipelex/tools/misc/json_utils.py", + "reference_start_line": 106, + "reference_end_line": 128 + }, + { + "qualified_name": "pipelex.utils.io.save_as_json_to_path", + "reference_file": "pipelex/tools/misc/json_utils.py", + "reference_start_line": 59, + "reference_end_line": 81 + } + ], + "assigned_files": [], + "can_expand": false + }, + { + "name": "Pipelex Pipeline", + "description": "Represents the actual pipelines within the Pipelex system, including their definition, structure, and execution flow. This component is responsible for the logical representation and operational aspects of a pipeline.", + "referenced_source_code": [ + { + "qualified_name": "pipelex.core.pipeline.execute_pipeline", + "reference_file": "pipelex/pipeline/execute.py", + "reference_start_line": 22, + "reference_end_line": 107 + }, + { + "qualified_name": "pipelex.core.pipeline.get_required_pipe", + "reference_file": "pipelex/core/pipes/pipe_abstract.py", + "reference_start_line": 41, + "reference_end_line": 49 + } + ], + "assigned_files": [], + "can_expand": true + }, + { + "name": "Unclassified", + "description": "Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies)", + "referenced_source_code": [], + "assigned_files": [], + "can_expand": false + } + ], + "components_relations": [ + { + "relation": "delegates to", + "src_name": "PipelexCLI", + "dst_name": "CLI Commands" + }, + { + "relation": "interacts with", + "src_name": "PipelexCLI", + "dst_name": "Pipelex System Configuration" + }, + { + "relation": "initiates operations on", + "src_name": "CLI Commands", + "dst_name": "Pipelex Core" + }, + { + "relation": "interacts with", + "src_name": "CLI Commands", + "dst_name": "Pipelex System Configuration" + }, + { + "relation": "manages", + "src_name": "Pipelex Core", + "dst_name": "Pipelex Pipeline" + }, + { + "relation": "interacts with", + "src_name": "Pipelex Core", + "dst_name": "Pipelex System Configuration" + } + ] +} diff --git a/.codeboarding/CLI_User_Interface.md b/.codeboarding/CLI_User_Interface.md new file mode 100644 index 000000000..072758b2e --- /dev/null +++ b/.codeboarding/CLI_User_Interface.md @@ -0,0 +1,81 @@ +```mermaid +graph LR + PipelexCLI["PipelexCLI"] + CLI_Commands["CLI Commands"] + Pipelex_Core["Pipelex Core"] + Pipelex_System_Configuration["Pipelex System Configuration"] + Pipelex_Pipeline["Pipelex Pipeline"] + Unclassified["Unclassified"] + PipelexCLI -- "delegates to" --> CLI_Commands + PipelexCLI -- "interacts with" --> Pipelex_System_Configuration + CLI_Commands -- "initiates operations on" --> Pipelex_Core + CLI_Commands -- "interacts with" --> Pipelex_System_Configuration + Pipelex_Core -- "manages" --> Pipelex_Pipeline + Pipelex_Core -- "interacts with" --> Pipelex_System_Configuration +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The Pipelex system's command-line interface is orchestrated by the PipelexCLI component, which acts as the central dispatcher for user commands. It delegates specific tasks to CLI Commands, such as run, build, and validate. These commands, in turn, interact with the Pipelex Core, which embodies the fundamental logic for pipeline execution, validation, and bundle management. The Pipelex Core relies on Pipelex System Configuration for managing settings and data, and directly manages Pipelex Pipeline instances. The Pipelex Builder component is responsible for generating pipelines based on user requirements, a function initiated by relevant CLI Commands. This architecture ensures a clear separation of concerns, with the CLI handling user interaction and command dispatch, while the core components manage the underlying pipeline logic and data. + +### PipelexCLI +Serves as the primary entry point for the command-line interface. It initializes the CLI application, registers available commands, parses user input, and dispatches control to the appropriate command handler. It acts as the orchestrator for user interactions with the Pipelex system. + + +**Related Classes/Methods**: + +- `pipelex.cli._cli.PipelexCLI`:14-27 + + +### CLI Commands +Each module within `pipelex.cli.commands` implements the specific business logic for a particular CLI command (e.g., run, build, validate). This includes handling command-line arguments, validating inputs, and invoking the relevant core Pipelex functionalities to perform tasks such as running, building, validating, or managing pipelines and concepts. + + +**Related Classes/Methods**: + +- `pipelex.cli.commands.run_cmd.run_cmd`:19-182 +- `pipelex.cli.commands.build_cmd.build_app` +- `pipelex.cli.commands.validate_cmd.validate_cmd`:33-148 + + +### Pipelex Core +Encapsulates the central functionalities of the Pipelex system, including pipeline execution, validation logic, and interaction with bundles. It provides the underlying mechanisms for processing and managing pipelines and their associated data. + + +**Related Classes/Methods**: + +- `pipelex.core.Pipelex`:64-292 +- `pipelex.core.interpreter.PipelexInterpreter`:17-92 + + +### Pipelex System Configuration +Manages the configuration settings and input/output data for the Pipelex system. This includes loading and saving JSON files for inputs and outputs, and potentially other system-wide settings. + + +**Related Classes/Methods**: + +- `pipelex.utils.io.load_json_dict_from_path`:106-128 +- `pipelex.utils.io.save_as_json_to_path`:59-81 + + +### Pipelex Pipeline +Represents the actual pipelines within the Pipelex system, including their definition, structure, and execution flow. This component is responsible for the logical representation and operational aspects of a pipeline. + + +**Related Classes/Methods**: + +- `pipelex.core.pipeline.execute_pipeline`:22-107 +- `pipelex.core.pipeline.get_required_pipe`:41-49 + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/.codeboarding/Reporting_Observability.json b/.codeboarding/Reporting_Observability.json new file mode 100644 index 000000000..86405ada6 --- /dev/null +++ b/.codeboarding/Reporting_Observability.json @@ -0,0 +1,93 @@ +{ + "description": "The Pipelex system's core execution is driven by the `Pipelex Core Execution Components`, which are responsible for building and running pipelines defined by `PipelexBundleSpec` objects. The `FlowFactory` constructs a high-level `Flow` representation, while `WorkingMemory` manages the runtime state. During this execution, the `LocalObserver` passively collects detailed event and metric data, storing it locally. This collected data is then consumed by the `ReportingManager`, which processes and aggregates it to generate comprehensive reports, particularly focusing on resource usage and costs associated with pipeline operations. This architecture ensures a clear separation of concerns between pipeline execution, data observation, and reporting.", + "components": [ + { + "name": "Pipelex Core Execution Components", + "description": "This logical grouping represents the operational parts of the Pipelex system responsible for building, interpreting, and executing pipelines. It includes the core logic for defining pipeline flows, managing pipe specifications, and handling working memory during execution. These components are the source of all observable events, metrics, and logs.", + "referenced_source_code": [ + { + "qualified_name": "pipelex.builder.builder.assemble_pipelex_bundle_spec", + "reference_file": "pipelex/builder/builder.py", + "reference_start_line": 53, + "reference_end_line": 123 + }, + { + "qualified_name": "pipelex.builder.flow.Flow", + "reference_file": "pipelex/builder/flow.py", + "reference_start_line": 31, + "reference_end_line": 54 + }, + { + "qualified_name": "pipelex.builder.flow_factory.FlowFactory", + "reference_file": "pipelex/builder/flow_factory.py", + "reference_start_line": 22, + "reference_end_line": 119 + }, + { + "qualified_name": "pipelex.core.memory.working_memory.WorkingMemory", + "reference_file": "pipelex/core/memory/working_memory.py", + "reference_start_line": 42, + "reference_end_line": 385 + } + ], + "assigned_files": [], + "can_expand": true + }, + { + "name": "LocalObserver", + "description": "The `LocalObserver` acts as the primary data collection agent within the subsystem. It actively monitors and captures specific events, performance metrics, and resource usage directly from the `Pipelex Core Execution Components` during pipeline execution. It writes these observations to local JSONL files.", + "referenced_source_code": [ + { + "qualified_name": "pipelex.observer.local_observer.LocalObserver", + "reference_file": "pipelex/observer/local_observer.py", + "reference_start_line": 10, + "reference_end_line": 35 + } + ], + "assigned_files": [ + "pipelex/observer/__init__.py", + "pipelex/observer/observer_protocol.py", + "pipelex/observer/multi_observer.py", + "pipelex/observer/local_observer.py" + ], + "can_expand": false + }, + { + "name": "ReportingManager", + "description": "The `ReportingManager` is the central hub for processing, aggregating, and presenting all collected observability data. It takes raw or preprocessed data from the `LocalObserver`, transforms it into meaningful insights, and manages the output of reports, particularly for LLM token usage and costs.", + "referenced_source_code": [ + { + "qualified_name": "pipelex.reporting.reporting_manager.ReportingManager", + "reference_file": "pipelex/reporting/reporting_manager.py", + "reference_start_line": 29, + "reference_end_line": 123 + } + ], + "assigned_files": [ + "pipelex/reporting/__init__.py", + "pipelex/reporting/reporting_manager.py", + "pipelex/reporting/reporting_protocol.py" + ], + "can_expand": true + }, + { + "name": "Unclassified", + "description": "Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies)", + "referenced_source_code": [], + "assigned_files": [], + "can_expand": false + } + ], + "components_relations": [ + { + "relation": "Exposes data to", + "src_name": "Pipelex Core Execution Components", + "dst_name": "LocalObserver" + }, + { + "relation": "Feeds data to", + "src_name": "LocalObserver", + "dst_name": "ReportingManager" + } + ] +} diff --git a/.codeboarding/Reporting_Observability.md b/.codeboarding/Reporting_Observability.md new file mode 100644 index 000000000..3c7b567c2 --- /dev/null +++ b/.codeboarding/Reporting_Observability.md @@ -0,0 +1,55 @@ +```mermaid +graph LR + Pipelex_Core_Execution_Components["Pipelex Core Execution Components"] + LocalObserver["LocalObserver"] + ReportingManager["ReportingManager"] + Unclassified["Unclassified"] + Pipelex_Core_Execution_Components -- "Exposes data to" --> LocalObserver + LocalObserver -- "Feeds data to" --> ReportingManager +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The Pipelex system's core execution is driven by the `Pipelex Core Execution Components`, which are responsible for building and running pipelines defined by `PipelexBundleSpec` objects. The `FlowFactory` constructs a high-level `Flow` representation, while `WorkingMemory` manages the runtime state. During this execution, the `LocalObserver` passively collects detailed event and metric data, storing it locally. This collected data is then consumed by the `ReportingManager`, which processes and aggregates it to generate comprehensive reports, particularly focusing on resource usage and costs associated with pipeline operations. This architecture ensures a clear separation of concerns between pipeline execution, data observation, and reporting. + +### Pipelex Core Execution Components +This logical grouping represents the operational parts of the Pipelex system responsible for building, interpreting, and executing pipelines. It includes the core logic for defining pipeline flows, managing pipe specifications, and handling working memory during execution. These components are the source of all observable events, metrics, and logs. + + +**Related Classes/Methods**: + +- `pipelex.builder.builder.assemble_pipelex_bundle_spec`:53-123 +- `pipelex.builder.flow.Flow`:31-54 +- `pipelex.builder.flow_factory.FlowFactory`:22-119 +- `pipelex.core.memory.working_memory.WorkingMemory`:42-385 + + +### LocalObserver +The `LocalObserver` acts as the primary data collection agent within the subsystem. It actively monitors and captures specific events, performance metrics, and resource usage directly from the `Pipelex Core Execution Components` during pipeline execution. It writes these observations to local JSONL files. + + +**Related Classes/Methods**: + +- `pipelex.observer.local_observer.LocalObserver`:10-35 + + +### ReportingManager +The `ReportingManager` is the central hub for processing, aggregating, and presenting all collected observability data. It takes raw or preprocessed data from the `LocalObserver`, transforms it into meaningful insights, and manages the output of reports, particularly for LLM token usage and costs. + + +**Related Classes/Methods**: + +- `pipelex.reporting.reporting_manager.ReportingManager`:29-123 + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/.codeboarding/analysis.json b/.codeboarding/analysis.json new file mode 100644 index 000000000..46a30db3c --- /dev/null +++ b/.codeboarding/analysis.json @@ -0,0 +1,588 @@ +{ + "description": "The Pipelex project is structured around a core orchestration engine that manages the execution of AI pipelines. User interaction primarily occurs through a Command-Line Interface (CLI), which initiates pipeline building and execution. Pipeline definitions, written in a declarative PLX format, are processed by a dedicated builder component to create an executable flow. During execution, the orchestration engine dispatches tasks to various pipe operators and controllers, which interact with AI inference and model management services for AI-driven tasks. Data and memory management components handle the flow and storage of information throughout the pipeline. Finally, a reporting and observability component tracks pipeline execution and generates reports.", + "components": [ + { + "name": "CLI & User Interface", + "description": "Provides the command-line interface for users to interact with Pipelex, enabling them to run, build, validate, and manage pipelines.", + "referenced_source_code": [ + { + "qualified_name": "pipelex.cli.PipelexCLI", + "reference_file": "pipelex/cli/_cli.py", + "reference_start_line": null, + "reference_end_line": null + }, + { + "qualified_name": "pipelex.cli.commands.run_cmd.run_cmd", + "reference_file": "pipelex/cli/commands/run_cmd.py", + "reference_start_line": 19, + "reference_end_line": 182 + } + ], + "assigned_files": [ + "pipelex/cli/_cli.py", + "pipelex/cli/__init__.py", + "pipelex/cli/commands/__init__.py", + "pipelex/cli/commands/build_cmd.py", + "pipelex/cli/commands/kit_cmd.py", + "pipelex/cli/commands/init_cmd.py", + "pipelex/cli/commands/validate_cmd.py", + "pipelex/cli/commands/run_cmd.py", + "pipelex/cli/commands/show_cmd.py" + ], + "can_expand": true + }, + { + "name": "Pipeline Definition & Builder", + "description": "Responsible for defining, parsing, and validating pipeline configurations (PLX files). It translates the declarative pipeline definitions into an executable flow graph.", + "referenced_source_code": [ + { + "qualified_name": "pipelex.builder.flow_factory.FlowFactory", + "reference_file": "pipelex/builder/flow_factory.py", + "reference_start_line": 22, + "reference_end_line": 119 + }, + { + "qualified_name": "pipelex.language.plx_factory.PlxFactory", + "reference_file": "pipelex/language/plx_factory.py", + "reference_start_line": 28, + "reference_end_line": 336 + } + ], + "assigned_files": [ + "pipelex/builder/builder.py", + "pipelex/builder/builder_validation.py", + "pipelex/builder/builder_loop.py", + "pipelex/builder/__init__.py", + "pipelex/builder/bundle_header_spec.py", + "pipelex/builder/runner_code.py", + "pipelex/builder/builder_errors.py", + "pipelex/builder/flow.py", + "pipelex/builder/bundle_spec.py", + "pipelex/builder/validation_error_data.py", + "pipelex/builder/pipe/pipe_spec_exceptions.py", + "pipelex/builder/pipe/sub_pipe_spec.py", + "pipelex/builder/pipe/pipe_parallel_spec.py", + "pipelex/builder/pipe/__init__.py", + "pipelex/builder/pipe/pipe_img_gen_spec.py", + "pipelex/builder/pipe/pipe_spec_union.py", + "pipelex/builder/pipe/pipe_compose_spec.py", + "pipelex/builder/pipe/pipe_spec.py", + "pipelex/builder/pipe/pipe_spec_map.py", + "pipelex/builder/pipe/pipe_llm_spec.py", + "pipelex/builder/pipe/pipe_extract_spec.py", + "pipelex/builder/pipe/pipe_batch_spec.py", + "pipelex/builder/pipe/pipe_condition_spec.py", + "pipelex/builder/pipe/pipe_sequence_spec.py", + "pipelex/builder/pipe/pipe_func_spec.py", + "pipelex/builder/pipe/pipe_signature.py", + "pipelex/builder/concept/__init__.py", + "pipelex/builder/concept/concept_spec_factory.py", + "pipelex/builder/concept/concept_spec.py", + "pipelex/language/plx_config.py", + "pipelex/language/__init__.py", + "pipelex/core/validation.py", + "pipelex/core/validation_errors.py", + "pipelex/core/pipes/__init__.py", + "pipelex/core/pipes/input_requirements_factory.py", + "pipelex/core/pipes/pipe_blueprint.py", + "pipelex/core/pipes/pipe_output.py", + "pipelex/core/pipes/variable_multiplicity.py", + "pipelex/core/pipes/pipe_library_abstract.py", + "pipelex/core/pipes/pipe_factory.py", + "pipelex/core/pipes/input_requirements.py", + "pipelex/core/pipes/pipe_abstract.py", + "pipelex/core/pipes/exceptions.py", + "pipelex/core/pipes/pipe_library.py", + "pipelex/core/domains/__init__.py", + "pipelex/core/domains/domain.py", + "pipelex/core/domains/domain_library.py", + "pipelex/core/domains/domain_library_abstract.py", + "pipelex/core/domains/domain_factory.py", + "pipelex/core/domains/exceptions.py", + "pipelex/core/domains/domain_blueprint.py", + "pipelex/core/bundles/__init__.py", + "pipelex/core/bundles/pipelex_bundle_blueprint.py", + "pipelex/core/bundles/pipe_sorter.py", + "pipelex/core/bundles/exceptions.py" + ], + "can_expand": true + }, + { + "name": "Core Orchestration Engine", + "description": "The central component that manages the entire lifecycle of a pipeline, from loading and execution to tracking and error handling. It orchestrates the flow of data and control between different pipe operators.", + "referenced_source_code": [ + { + "qualified_name": "pipelex.Pipelex", + "reference_file": "pipelex/pipelex.py", + "reference_start_line": 64, + "reference_end_line": 370 + }, + { + "qualified_name": "pipelex.pipeline.pipeline_manager.PipelineManager", + "reference_file": "pipelex/pipeline/pipeline_manager.py", + "reference_start_line": 12, + "reference_end_line": 43 + } + ], + "assigned_files": [ + "pipelex/pipelex.py", + "pipelex/hub.py", + "pipelex/pipeline/pipeline_models.py", + "pipelex/pipeline/__init__.py", + "pipelex/pipeline/pipeline_manager_abstract.py", + "pipelex/pipeline/job_metadata.py", + "pipelex/pipeline/pipeline_manager.py", + "pipelex/pipeline/start.py", + "pipelex/pipeline/pipeline_factory.py", + "pipelex/pipeline/pipeline.py", + "pipelex/pipeline/execute.py", + "pipelex/pipeline/track/__init__.py", + "pipelex/pipeline/track/pipeline_tracker_protocol.py", + "pipelex/pipeline/track/tracker_models.py", + "pipelex/pipeline/track/pipeline_tracker.py", + "pipelex/pipeline/track/flow_chart.py", + "pipelex/pipeline/track/tracker_config.py", + "pipelex/pipe_run/__init__.py", + "pipelex/pipe_run/pipe_run_params.py", + "pipelex/pipe_run/pipe_router.py", + "pipelex/pipe_run/pipe_job.py", + "pipelex/pipe_run/dry_run.py", + "pipelex/pipe_run/pipe_router_protocol.py", + "pipelex/pipe_run/pipe_run_params_factory.py", + "pipelex/pipe_run/pipe_job_factory.py", + "pipelex/core/pipe_errors.py", + "pipelex/core/interpreter.py", + "pipelex/core/exceptions.py", + "pipelex/core/registry_models.py" + ], + "can_expand": true + }, + { + "name": "Pipe Operators & Controllers", + "description": "This component group encompasses the individual processing units (operators) and control flow mechanisms (controllers) within a pipeline. Operators perform specific AI-driven tasks, while controllers manage the execution logic.", + "referenced_source_code": [ + { + "qualified_name": "pipelex.pipe_operators.pipe_operator.PipeOperator", + "reference_file": "pipelex/pipe_operators/pipe_operator.py", + "reference_start_line": 17, + "reference_end_line": 94 + }, + { + "qualified_name": "pipelex.pipe_controllers.pipe_controller.PipeController", + "reference_file": "pipelex/pipe_controllers/pipe_controller.py", + "reference_start_line": 14, + "reference_end_line": 86 + } + ], + "assigned_files": [ + "pipelex/pipe_controllers/__init__.py", + "pipelex/pipe_controllers/sub_pipe_blueprint.py", + "pipelex/pipe_controllers/pipe_controller.py", + "pipelex/pipe_controllers/sub_pipe.py", + "pipelex/pipe_controllers/sub_pipe_factory.py", + "pipelex/pipe_controllers/parallel/__init__.py", + "pipelex/pipe_controllers/parallel/pipe_parallel_blueprint.py", + "pipelex/pipe_controllers/parallel/pipe_parallel.py", + "pipelex/pipe_controllers/parallel/pipe_parallel_factory.py", + "pipelex/pipe_controllers/condition/__init__.py", + "pipelex/pipe_controllers/condition/pipe_condition_details.py", + "pipelex/pipe_controllers/condition/pipe_condition.py", + "pipelex/pipe_controllers/condition/pipe_condition_blueprint.py", + "pipelex/pipe_controllers/condition/special_outcome.py", + "pipelex/pipe_controllers/condition/pipe_condition_factory.py", + "pipelex/pipe_controllers/sequence/__init__.py", + "pipelex/pipe_controllers/sequence/pipe_sequence.py", + "pipelex/pipe_controllers/sequence/pipe_sequence_factory.py", + "pipelex/pipe_controllers/sequence/exceptions.py", + "pipelex/pipe_controllers/sequence/pipe_sequence_blueprint.py", + "pipelex/pipe_controllers/batch/__init__.py", + "pipelex/pipe_controllers/batch/pipe_batch.py", + "pipelex/pipe_controllers/batch/pipe_batch_factory.py", + "pipelex/pipe_controllers/batch/pipe_batch_blueprint.py", + "pipelex/pipe_operators/__init__.py", + "pipelex/pipe_operators/func/__init__.py", + "pipelex/pipe_operators/func/pipe_func_blueprint.py", + "pipelex/pipe_operators/func/pipe_func_factory.py", + "pipelex/pipe_operators/func/pipe_func.py", + "pipelex/pipe_operators/compose/__init__.py", + "pipelex/pipe_operators/compose/pipe_compose.py", + "pipelex/pipe_operators/compose/pipe_compose_factory.py", + "pipelex/pipe_operators/compose/pipe_compose_blueprint.py", + "pipelex/pipe_operators/extract/__init__.py", + "pipelex/pipe_operators/extract/pipe_extract.py", + "pipelex/pipe_operators/extract/pipe_extract_blueprint.py", + "pipelex/pipe_operators/extract/pipe_extract_factory.py", + "pipelex/pipe_operators/llm/__init__.py", + "pipelex/pipe_operators/llm/pipe_llm_factory.py", + "pipelex/pipe_operators/llm/llm_prompt_blueprint.py", + "pipelex/pipe_operators/llm/pipe_llm_blueprint.py", + "pipelex/pipe_operators/llm/pipe_llm.py", + "pipelex/pipe_operators/img_gen/__init__.py", + "pipelex/pipe_operators/img_gen/pipe_img_gen_factory.py", + "pipelex/pipe_operators/img_gen/pipe_img_gen_blueprint.py", + "pipelex/pipe_operators/img_gen/pipe_img_gen.py" + ], + "can_expand": true + }, + { + "name": "AI Inference & Model Management", + "description": "Provides a standardized interface for interacting with various external AI models and inference services. It abstracts away the complexities of different AI provider APIs and manages model configurations.", + "referenced_source_code": [ + { + "qualified_name": "pipelex.cogt.inference.inference_manager.InferenceManager", + "reference_file": "pipelex/cogt/inference/inference_manager.py", + "reference_start_line": 18, + "reference_end_line": 150 + }, + { + "qualified_name": "pipelex.cogt.models.model_manager.ModelManager", + "reference_file": "pipelex/cogt/models/model_manager.py", + "reference_start_line": 20, + "reference_end_line": 148 + } + ], + "assigned_files": [ + "pipelex/cogt/__init__.py", + "pipelex/cogt/config_cogt.py", + "pipelex/cogt/exceptions.py", + "pipelex/cogt/inference/__init__.py", + "pipelex/cogt/inference/inference_job_abstract.py", + "pipelex/cogt/inference/inference_manager_protocol.py", + "pipelex/cogt/inference/inference_manager.py", + "pipelex/cogt/inference/inference_worker_abstract.py", + "pipelex/cogt/model_backends/backend.py", + "pipelex/cogt/model_backends/backend_factory.py", + "pipelex/cogt/model_backends/__init__.py", + "pipelex/cogt/model_backends/model_constraints.py", + "pipelex/cogt/model_backends/model_spec_factory.py", + "pipelex/cogt/model_backends/prompting_target.py", + "pipelex/cogt/model_backends/model_lists.py", + "pipelex/cogt/model_backends/backend_library.py", + "pipelex/cogt/model_backends/model_spec.py", + "pipelex/cogt/model_backends/model_type.py", + "pipelex/cogt/extract/extract_job_components.py", + "pipelex/cogt/extract/__init__.py", + "pipelex/cogt/extract/extract_setting.py", + "pipelex/cogt/extract/extract_job.py", + "pipelex/cogt/extract/extract_job_factory.py", + "pipelex/cogt/extract/extract_input.py", + "pipelex/cogt/extract/extract_worker_abstract.py", + "pipelex/cogt/extract/extract_output.py", + "pipelex/cogt/extract/extract_worker_factory.py", + "pipelex/cogt/templating/__init__.py", + "pipelex/cogt/templating/template_rendering.py", + "pipelex/cogt/templating/template_preprocessor.py", + "pipelex/cogt/templating/template_blueprint.py", + "pipelex/cogt/templating/template_category.py", + "pipelex/cogt/templating/templating_style.py", + "pipelex/cogt/image/prompt_image.py", + "pipelex/cogt/image/__init__.py", + "pipelex/cogt/image/prompt_image_factory.py", + "pipelex/cogt/image/generated_image.py", + "pipelex/cogt/usage/__init__.py", + "pipelex/cogt/usage/costs_per_token.py", + "pipelex/cogt/usage/cost_registry.py", + "pipelex/cogt/usage/token_category.py", + "pipelex/cogt/usage/cost_category.py", + "pipelex/cogt/llm/llm_job.py", + "pipelex/cogt/llm/llm_job_components.py", + "pipelex/cogt/llm/llm_prompt_template.py", + "pipelex/cogt/llm/__init__.py", + "pipelex/cogt/llm/llm_utils.py", + "pipelex/cogt/llm/llm_worker_abstract.py", + "pipelex/cogt/llm/llm_setting.py", + "pipelex/cogt/llm/llm_worker_factory.py", + "pipelex/cogt/llm/llm_worker_internal_abstract.py", + "pipelex/cogt/llm/llm_job_factory.py", + "pipelex/cogt/llm/llm_prompt_factory_abstract.py", + "pipelex/cogt/llm/llm_report.py", + "pipelex/cogt/llm/llm_prompt.py", + "pipelex/cogt/llm/structured_output.py", + "pipelex/cogt/llm/llm_prompt_template_inputs.py", + "pipelex/cogt/model_routing/routing_models.py", + "pipelex/cogt/model_routing/__init__.py", + "pipelex/cogt/model_routing/routing_profile_factory.py", + "pipelex/cogt/model_routing/routing_profile_library.py", + "pipelex/cogt/model_routing/routing_profile.py", + "pipelex/cogt/content_generation/__init__.py", + "pipelex/cogt/content_generation/content_generator_dry.py", + "pipelex/cogt/content_generation/assignment_models.py", + "pipelex/cogt/content_generation/llm_generate.py", + "pipelex/cogt/content_generation/templating_generate.py", + "pipelex/cogt/content_generation/extract_generate.py", + "pipelex/cogt/content_generation/content_generator_protocol.py", + "pipelex/cogt/content_generation/content_generator.py", + "pipelex/cogt/content_generation/img_gen_generate.py", + "pipelex/cogt/img_gen/img_gen_job_components.py", + "pipelex/cogt/img_gen/__init__.py", + "pipelex/cogt/img_gen/img_gen_worker_abstract.py", + "pipelex/cogt/img_gen/img_gen_job.py", + "pipelex/cogt/img_gen/img_gen_worker_factory.py", + "pipelex/cogt/img_gen/img_gen_setting.py", + "pipelex/cogt/img_gen/img_gen_job_factory.py", + "pipelex/cogt/img_gen/img_gen_prompt.py", + "pipelex/cogt/models/__init__.py", + "pipelex/cogt/models/model_manager_abstract.py", + "pipelex/cogt/models/model_manager.py", + "pipelex/cogt/models/model_deck_check.py", + "pipelex/cogt/models/model_deck.py", + "pipelex/plugins/__init__.py", + "pipelex/plugins/plugin_sdk_registry.py", + "pipelex/plugins/plugin_manager.py", + "pipelex/plugins/openai/__init__.py", + "pipelex/plugins/openai/openai_llm_worker.py", + "pipelex/plugins/openai/openai_llms.py", + "pipelex/plugins/openai/openai_factory.py", + "pipelex/plugins/openai/openai_func.py", + "pipelex/plugins/openai/vertexai_factory.py", + "pipelex/plugins/openai/openai_img_gen_worker.py", + "pipelex/plugins/openai/openai_img_gen_factory.py", + "pipelex/plugins/bedrock/bedrock_llms.py", + "pipelex/plugins/bedrock/__init__.py", + "pipelex/plugins/bedrock/bedrock_client_protocol.py", + "pipelex/plugins/bedrock/bedrock_factory.py", + "pipelex/plugins/bedrock/bedrock_message.py", + "pipelex/plugins/bedrock/bedrock_llm_worker.py", + "pipelex/plugins/bedrock/bedrock_client_aioboto3.py", + "pipelex/plugins/bedrock/bedrock_client_boto3.py", + "pipelex/plugins/anthropic/__init__.py", + "pipelex/plugins/anthropic/anthropic_exceptions.py", + "pipelex/plugins/anthropic/anthropic_llm_worker.py", + "pipelex/plugins/anthropic/anthropic_factory.py", + "pipelex/plugins/anthropic/anthropic_llms.py", + "pipelex/plugins/fal/__init__.py", + "pipelex/plugins/fal/fal_config.py", + "pipelex/plugins/fal/fal_factory.py", + "pipelex/plugins/fal/fal_img_gen_worker.py", + "pipelex/plugins/pypdfium2/__init__.py", + "pipelex/plugins/pypdfium2/pypdfium2_worker.py", + "pipelex/plugins/google/__init__.py", + "pipelex/plugins/google/google_factory.py", + "pipelex/plugins/google/google_llm_worker.py", + "pipelex/plugins/mistral/__init__.py", + "pipelex/plugins/mistral/mistral_factory.py", + "pipelex/plugins/mistral/mistral_exceptions.py", + "pipelex/plugins/mistral/mistral_llms.py", + "pipelex/plugins/mistral/mistral_llm_worker.py", + "pipelex/plugins/mistral/mistral_extract_worker.py" + ], + "can_expand": true + }, + { + "name": "Data & Memory Management", + "description": "Manages the flow, storage, and retrieval of data (\"stuffs\" and \"concepts\") within a running pipeline. This includes inputs, intermediate results, and final outputs.", + "referenced_source_code": [ + { + "qualified_name": "pipelex.core.memory.working_memory.WorkingMemory", + "reference_file": "pipelex/core/memory/working_memory.py", + "reference_start_line": 42, + "reference_end_line": 385 + }, + { + "qualified_name": "pipelex.core.stuffs.stuff_factory.StuffFactory", + "reference_file": "pipelex/core/stuffs/stuff_factory.py", + "reference_start_line": 38, + "reference_end_line": 487 + } + ], + "assigned_files": [ + "pipelex/core/stuffs/text_and_images_content.py", + "pipelex/core/stuffs/__init__.py", + "pipelex/core/stuffs/image_field_search.py", + "pipelex/core/stuffs/list_content.py", + "pipelex/core/stuffs/stuff_factory.py", + "pipelex/core/stuffs/stuff_content.py", + "pipelex/core/stuffs/stuff_artefact.py", + "pipelex/core/stuffs/pdf_content.py", + "pipelex/core/stuffs/text_content.py", + "pipelex/core/stuffs/stuff.py", + "pipelex/core/stuffs/number_content.py", + "pipelex/core/stuffs/mermaid_content.py", + "pipelex/core/stuffs/page_content.py", + "pipelex/core/stuffs/json_content.py", + "pipelex/core/stuffs/dynamic_content.py", + "pipelex/core/stuffs/image_content.py", + "pipelex/core/stuffs/structured_content.py", + "pipelex/core/stuffs/html_content.py", + "pipelex/core/memory/__init__.py", + "pipelex/core/memory/working_memory.py", + "pipelex/core/memory/working_memory_factory.py", + "pipelex/core/concepts/__init__.py", + "pipelex/core/concepts/structure_generator.py", + "pipelex/core/concepts/concept_blueprint.py", + "pipelex/core/concepts/concept.py", + "pipelex/core/concepts/concept_library.py", + "pipelex/core/concepts/concept_native.py", + "pipelex/core/concepts/exceptions.py", + "pipelex/core/concepts/concept_factory.py", + "pipelex/core/concepts/concept_library_abstract.py" + ], + "can_expand": true + }, + { + "name": "Reporting & Observability", + "description": "Gathers and presents metrics and logs related to pipeline execution, offering insights into performance, resource usage, and potential issues.", + "referenced_source_code": [ + { + "qualified_name": "pipelex.reporting.reporting_manager.ReportingManager", + "reference_file": "pipelex/reporting/reporting_manager.py", + "reference_start_line": 29, + "reference_end_line": 123 + }, + { + "qualified_name": "pipelex.observer.local_observer.LocalObserver", + "reference_file": "pipelex/observer/local_observer.py", + "reference_start_line": 10, + "reference_end_line": 35 + } + ], + "assigned_files": [ + "pipelex/observer/__init__.py", + "pipelex/observer/observer_protocol.py", + "pipelex/observer/multi_observer.py", + "pipelex/observer/local_observer.py", + "pipelex/reporting/__init__.py", + "pipelex/reporting/reporting_manager.py", + "pipelex/reporting/reporting_protocol.py" + ], + "can_expand": true + }, + { + "name": "Unclassified", + "description": "Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies)", + "referenced_source_code": [], + "assigned_files": [ + "pipelex/__init__.py", + "pipelex/types.py", + "pipelex/urls.py", + "pipelex/exceptions.py", + "pipelex/config.py", + "pipelex/tools/__init__.py", + "pipelex/tools/secrets/__init__.py", + "pipelex/tools/secrets/env_secrets_provider.py", + "pipelex/tools/secrets/secrets_errors.py", + "pipelex/tools/secrets/secrets_utils.py", + "pipelex/tools/secrets/secrets_provider_abstract.py", + "pipelex/tools/misc/__init__.py", + "pipelex/tools/misc/context_provider_abstract.py", + "pipelex/tools/misc/attribute_utils.py", + "pipelex/tools/misc/base_64_utils.py", + "pipelex/tools/misc/path_utils.py", + "pipelex/tools/misc/file_fetch_utils.py", + "pipelex/tools/misc/pretty.py", + "pipelex/tools/misc/toml_utils.py", + "pipelex/tools/misc/filetype_utils.py", + "pipelex/tools/misc/placeholder.py", + "pipelex/tools/misc/file_utils.py", + "pipelex/tools/misc/terminal_utils.py", + "pipelex/tools/misc/string_utils.py", + "pipelex/tools/misc/dict_utils.py", + "pipelex/tools/misc/json_utils.py", + "pipelex/tools/misc/markdown_utils.py", + "pipelex/tools/misc/mermaid_utils.py", + "pipelex/tools/aws/__init__.py", + "pipelex/tools/aws/aws_config.py", + "pipelex/tools/storage/__init__.py", + "pipelex/tools/storage/storage_provider_abstract.py", + "pipelex/tools/log/__init__.py", + "pipelex/tools/log/log_config.py", + "pipelex/tools/log/log_formatter.py", + "pipelex/tools/log/log_levels.py", + "pipelex/tools/log/log_dispatch.py", + "pipelex/tools/log/log.py", + "pipelex/tools/typing/validation_utils.py", + "pipelex/tools/typing/__init__.py", + "pipelex/tools/typing/module_inspector.py", + "pipelex/tools/typing/structure_printer.py", + "pipelex/tools/typing/class_utils.py", + "pipelex/tools/typing/pydantic_utils.py", + "pipelex/tools/pdf/__init__.py", + "pipelex/tools/pdf/pypdfium2_renderer.py", + "pipelex/tools/jinja2/jinja2_required_variables.py", + "pipelex/tools/jinja2/__init__.py", + "pipelex/tools/jinja2/jinja2_models.py", + "pipelex/tools/jinja2/jinja2_errors.py", + "pipelex/tools/jinja2/jinja2_rendering.py", + "pipelex/tools/jinja2/jinja2_environment.py", + "pipelex/tools/jinja2/jinja2_parsing.py", + "pipelex/tools/jinja2/jinja2_filters.py", + "pipelex/system/__init__.py", + "pipelex/system/environment.py", + "pipelex/system/runtime.py", + "pipelex/system/exceptions.py", + "pipelex/system/configuration/config_loader.py", + "pipelex/system/configuration/__init__.py", + "pipelex/system/configuration/config_model.py", + "pipelex/system/configuration/config_root.py", + "pipelex/system/registries/__init__.py", + "pipelex/system/registries/func_registry.py", + "pipelex/system/registries/registry_base.py", + "pipelex/system/registries/class_registry_utils.py", + "pipelex/system/registries/func_registry_utils.py", + "pipelex/client/__init__.py", + "pipelex/client/pipeline_request_factory.py", + "pipelex/client/api_serializer.py", + "pipelex/client/pipeline_response_factory.py", + "pipelex/client/client.py", + "pipelex/client/protocol.py", + "pipelex/core/__init__.py", + "pipelex/libraries/__init__.py", + "pipelex/libraries/library_manager.py", + "pipelex/libraries/library_utils.py", + "pipelex/libraries/library_manager_abstract.py", + "pipelex/libraries/library_manager_factory.py", + "pipelex/kit/paths.py", + "pipelex/kit/index_models.py", + "pipelex/kit/__init__.py", + "pipelex/kit/index_loader.py", + "pipelex/kit/markers.py", + "pipelex/kit/migrations_export.py", + "pipelex/kit/targets_update.py", + "pipelex/kit/cursor_export.py" + ], + "can_expand": false + } + ], + "components_relations": [ + { + "relation": "initiates", + "src_name": "CLI & User Interface", + "dst_name": "Pipeline Definition & Builder" + }, + { + "relation": "initiates", + "src_name": "CLI & User Interface", + "dst_name": "Core Orchestration Engine" + }, + { + "relation": "outputs pipeline configuration to", + "src_name": "Pipeline Definition & Builder", + "dst_name": "Core Orchestration Engine" + }, + { + "relation": "orchestrates", + "src_name": "Core Orchestration Engine", + "dst_name": "Pipe Operators & Controllers" + }, + { + "relation": "interacts with", + "src_name": "Pipe Operators & Controllers", + "dst_name": "AI Inference & Model Management" + }, + { + "relation": "reads from/writes to", + "src_name": "Pipe Operators & Controllers", + "dst_name": "Data & Memory Management" + }, + { + "relation": "interacts with", + "src_name": "Core Orchestration Engine", + "dst_name": "Data & Memory Management" + }, + { + "relation": "sends events to", + "src_name": "Core Orchestration Engine", + "dst_name": "Reporting & Observability" + } + ] +} diff --git a/.codeboarding/codeboarding_version.json b/.codeboarding/codeboarding_version.json new file mode 100644 index 000000000..3ba21c81d --- /dev/null +++ b/.codeboarding/codeboarding_version.json @@ -0,0 +1,4 @@ +{ + "commit_hash": "564390982c7da6cb6a93d157e111692ea04d3993", + "code_boarding_version": "0.2.0" +} diff --git a/.codeboarding/overview.md b/.codeboarding/overview.md new file mode 100644 index 000000000..a7159d371 --- /dev/null +++ b/.codeboarding/overview.md @@ -0,0 +1,107 @@ +```mermaid +graph LR + CLI_User_Interface["CLI & User Interface"] + Pipeline_Definition_Builder["Pipeline Definition & Builder"] + Core_Orchestration_Engine["Core Orchestration Engine"] + Pipe_Operators_Controllers["Pipe Operators & Controllers"] + AI_Inference_Model_Management["AI Inference & Model Management"] + Data_Memory_Management["Data & Memory Management"] + Reporting_Observability["Reporting & Observability"] + Unclassified["Unclassified"] + CLI_User_Interface -- "initiates" --> Pipeline_Definition_Builder + CLI_User_Interface -- "initiates" --> Core_Orchestration_Engine + Pipeline_Definition_Builder -- "outputs pipeline configuration to" --> Core_Orchestration_Engine + Core_Orchestration_Engine -- "orchestrates" --> Pipe_Operators_Controllers + Pipe_Operators_Controllers -- "interacts with" --> AI_Inference_Model_Management + Pipe_Operators_Controllers -- "reads from/writes to" --> Data_Memory_Management + Core_Orchestration_Engine -- "interacts with" --> Data_Memory_Management + Core_Orchestration_Engine -- "sends events to" --> Reporting_Observability + click CLI_User_Interface href "https://github.com/Pipelex/pipelex/blob/main/.codeboarding/CLI_User_Interface.md" "Details" + click Reporting_Observability href "https://github.com/Pipelex/pipelex/blob/main/.codeboarding/Reporting_Observability.md" "Details" +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The Pipelex project is structured around a core orchestration engine that manages the execution of AI pipelines. User interaction primarily occurs through a Command-Line Interface (CLI), which initiates pipeline building and execution. Pipeline definitions, written in a declarative PLX format, are processed by a dedicated builder component to create an executable flow. During execution, the orchestration engine dispatches tasks to various pipe operators and controllers, which interact with AI inference and model management services for AI-driven tasks. Data and memory management components handle the flow and storage of information throughout the pipeline. Finally, a reporting and observability component tracks pipeline execution and generates reports. + +### CLI & User Interface [[Expand]](./CLI_User_Interface.md) +Provides the command-line interface for users to interact with Pipelex, enabling them to run, build, validate, and manage pipelines. + + +**Related Classes/Methods**: + +- `pipelex.cli.PipelexCLI` +- `pipelex.cli.commands.run_cmd.run_cmd`:19-182 + + +### Pipeline Definition & Builder +Responsible for defining, parsing, and validating pipeline configurations (PLX files). It translates the declarative pipeline definitions into an executable flow graph. + + +**Related Classes/Methods**: + +- `pipelex.builder.flow_factory.FlowFactory`:22-119 +- `pipelex.language.plx_factory.PlxFactory`:28-336 + + +### Core Orchestration Engine +The central component that manages the entire lifecycle of a pipeline, from loading and execution to tracking and error handling. It orchestrates the flow of data and control between different pipe operators. + + +**Related Classes/Methods**: + +- `pipelex.Pipelex`:64-370 +- `pipelex.pipeline.pipeline_manager.PipelineManager`:12-43 + + +### Pipe Operators & Controllers +This component group encompasses the individual processing units (operators) and control flow mechanisms (controllers) within a pipeline. Operators perform specific AI-driven tasks, while controllers manage the execution logic. + + +**Related Classes/Methods**: + +- `pipelex.pipe_operators.pipe_operator.PipeOperator`:17-94 +- `pipelex.pipe_controllers.pipe_controller.PipeController`:14-86 + + +### AI Inference & Model Management +Provides a standardized interface for interacting with various external AI models and inference services. It abstracts away the complexities of different AI provider APIs and manages model configurations. + + +**Related Classes/Methods**: + +- `pipelex.cogt.inference.inference_manager.InferenceManager`:18-150 +- `pipelex.cogt.models.model_manager.ModelManager`:20-148 + + +### Data & Memory Management +Manages the flow, storage, and retrieval of data ("stuffs" and "concepts") within a running pipeline. This includes inputs, intermediate results, and final outputs. + + +**Related Classes/Methods**: + +- `pipelex.core.memory.working_memory.WorkingMemory`:42-385 +- `pipelex.core.stuffs.stuff_factory.StuffFactory`:38-487 + + +### Reporting & Observability [[Expand]](./Reporting_Observability.md) +Gathers and presents metrics and logs related to pipeline execution, offering insights into performance, resource usage, and potential issues. + + +**Related Classes/Methods**: + +- `pipelex.reporting.reporting_manager.ReportingManager`:29-123 +- `pipelex.observer.local_observer.LocalObserver`:10-35 + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/.github/workflows/docs-update.yml b/.github/workflows/docs-update.yml new file mode 100644 index 000000000..0cf29643e --- /dev/null +++ b/.github/workflows/docs-update.yml @@ -0,0 +1,156 @@ +name: CodeBoarding Documentation update workflow + +on: + schedule: + - cron: '0 20 * * 6' # Every Saturday at 8:00 PM UTC + workflow_dispatch: + inputs: + repository_url: + description: 'Repository URL to test with' + required: false + default: 'https://github.com/Pipelex/pipelex' + type: string + source_branch: + description: 'Source branch for generation' + required: false + default: 'main' + type: string + target_branch: + description: 'Target branch for pull request' + required: false + default: 'main' + type: string + output_format: + description: 'Output format for documentation' + required: false + default: '.md' + type: choice + options: + - '.mdx' + - '.md' + - '.rst' + output_directory: + description: 'Output directory for documentation files' + required: false + default: '.codeboarding' + type: string + +jobs: + update-docs-action-usage: + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: write + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 0 # Required to access branch history + + # Determine branches based on context + - name: Set branch variables + id: set-branches + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "source_branch=${{ github.head_ref }}" >> $GITHUB_OUTPUT + echo "target_branch=${{ github.base_ref }}" >> $GITHUB_OUTPUT + elif [ "${{ github.event.inputs.source_branch }}" != "" ] && [ "${{ github.event.inputs.target_branch }}" != "" ]; then + echo "source_branch=${{ github.event.inputs.source_branch }}" >> $GITHUB_OUTPUT + echo "target_branch=${{ github.event.inputs.target_branch }}" >> $GITHUB_OUTPUT + else + echo "source_branch=main" >> $GITHUB_OUTPUT + echo "target_branch=main" >> $GITHUB_OUTPUT + fi + + - name: Fetch CodeBoarding Documentation + timeout-minutes: 30 + id: codeboarding + uses: CodeBoarding/CodeBoarding-GHAction@0.1.2 + with: + repository_url: ${{ github.event.inputs.repository_url || 'https://github.com/Pipelex/pipelex' }} + source_branch: ${{ steps.set-branches.outputs.source_branch }} + target_branch: ${{ steps.set-branches.outputs.target_branch }} + output_directory: ${{ github.event.inputs.output_directory || '.codeboarding' }} + output_format: ${{ github.event.inputs.output_format || '.md' }} + + - name: Display Action Results + run: | + echo "Documentation files created: ${{ steps.codeboarding.outputs.markdown_files_created }}" + echo "JSON files created: ${{ steps.codeboarding.outputs.json_files_created }}" + echo "Documentation directory: ${{ steps.codeboarding.outputs.output_directory }}" + echo "JSON directory: ${{ steps.codeboarding.outputs.json_directory }}" + echo "Has changes: ${{ steps.codeboarding.outputs.has_changes }}" + + # Check if we have any changes to commit + - name: Check for changes + id: git-changes + run: | + if [ -n "$(git status --porcelain)" ]; then + echo "has_git_changes=true" >> $GITHUB_OUTPUT + else + echo "has_git_changes=false" >> $GITHUB_OUTPUT + fi + + # Copy CodeBoarding files to docs/pages/advanced-customization + - name: Copy CodeBoarding documentation to advanced-customization + if: steps.git-changes.outputs.has_git_changes == 'true' && steps.codeboarding.outputs.has_changes == 'true' + run: | + # Create docs/pages/advanced-customization directory if it doesn't exist + mkdir -p docs/pages/advanced-customization + + # Log the files found in the CodeBoarding directory + echo "📁 Scanning CodeBoarding directory for .md files..." + ls -la .codeboarding/ || echo "⚠️ CodeBoarding directory not found" + + # Copy all .md files from CodeBoarding to advanced-customization + copied_files_count=0 + + for file in .codeboarding/*.md; do + if [ -f "$file" ]; then + filename=$(basename "$file") + echo "✅ Copying: $filename to docs/pages/advanced-customization/" + cp "$file" "docs/pages/advanced-customization/$filename" + copied_files_count=$((copied_files_count + 1)) + fi + done + + # Summary logging + echo "" + echo "📊 File copy summary:" + echo " - Total .md files copied: $copied_files_count" + echo " - Destination: docs/pages/advanced-customization/" + + # List final contents + if [ $copied_files_count -gt 0 ]; then + echo " - Files in destination:" + ls -la docs/pages/advanced-customization/*.md 2>/dev/null || echo " No .md files found in destination" + fi + + echo "CodeBoarding documentation copied to docs/pages/advanced-customization/" + + - name: Commit and push changes + if: steps.git-changes.outputs.has_git_changes == 'true' && steps.codeboarding.outputs.has_changes == 'true' + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git add . + git commit -m "docs: update codeboarding documentation in advanced-customization + + ## 📚 Documentation Update + This commit contains updated documentation files fetched from the CodeBoarding service and copied to the advanced-customization section. + + ### 📊 Summary + - Documentation files created/updated: ${{ steps.codeboarding.outputs.markdown_files_created }} + - JSON files created/updated: ${{ steps.codeboarding.outputs.json_files_created }} + - Documentation directory: ${{ steps.codeboarding.outputs.output_directory }}/ + - JSON directory: ${{ steps.codeboarding.outputs.json_directory }}/ + - Output format: ${{ github.event.inputs.output_format || '.md' }} + - Repository analyzed: ${{ steps.codeboarding.outputs.repo_url }} + - Destination: docs/pages/advanced-customization/ + + The generated .md files have been automatically copied to the advanced-customization documentation section. + + 🤖 This commit was automatically generated by the CodeBoarding documentation update workflow." + git push diff --git a/docs/pages/advanced-customization/AI_Integration_Layer.md b/docs/pages/advanced-customization/AI_Integration_Layer.md new file mode 100644 index 000000000..1b9756fd1 --- /dev/null +++ b/docs/pages/advanced-customization/AI_Integration_Layer.md @@ -0,0 +1,100 @@ +```mermaid +graph LR + InferenceManager["InferenceManager"] + LLMWorker["LLMWorker"] + ImggWorker["ImggWorker"] + OcrWorker["OcrWorker"] + LLMDeck["LLMDeck"] + LLMModelLibrary["LLMModelLibrary"] + AI_Configuration["AI Configuration"] + InferenceManager -- "orchestrates" --> LLMWorker + InferenceManager -- "orchestrates" --> ImggWorker + InferenceManager -- "orchestrates" --> OcrWorker + InferenceManager -- "is configured by" --> AI_Configuration + LLMWorker -- "receives tasks from" --> InferenceManager + LLMWorker -- "uses configurations from" --> LLMDeck + LLMWorker -- "uses configurations from" --> LLMModelLibrary + ImggWorker -- "receives tasks from" --> InferenceManager + ImggWorker -- "is configured by" --> AI_Configuration + OcrWorker -- "receives tasks from" --> InferenceManager + OcrWorker -- "is configured by" --> AI_Configuration + LLMDeck -- "provides configurations to" --> LLMWorker + LLMModelLibrary -- "provides models to" --> LLMWorker + AI_Configuration -- "configures" --> InferenceManager + AI_Configuration -- "configures" --> LLMWorker + AI_Configuration -- "configures" --> ImggWorker + AI_Configuration -- "configures" --> OcrWorker +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/GeneratedOnBoardings)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/demo)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The AI Integration Layer in pipelex provides a unified interface to various AI service providers, abstracting away their specific APIs, and is utilized by the Pipe Operators for AI-related tasks. This layer is primarily encapsulated within the pipelex.cogt package. + +### InferenceManager +The central orchestrator for all AI inference tasks (LLM, Image Generation, OCR). It receives inference requests and dispatches them to the appropriate specialized AI workers. + + +**Related Classes/Methods**: + +- `pipelex.cogt.inference.inference_manager` + + +### LLMWorker +Defines the abstract interface for interacting with various Large Language Model (LLM) providers. Concrete implementations (e.g., Anthropic, OpenAI) handle provider-specific API calls. + + +**Related Classes/Methods**: + +- `pipelex.cogt.llm.llm_worker_abstract` + + +### ImggWorker +Defines the abstract interface for interacting with different Image Generation (Imgg) providers. Concrete implementations handle provider-specific API calls. + + +**Related Classes/Methods**: + +- `pipelex.cogt.imgg.imgg_worker_abstract` + + +### OcrWorker +Defines the abstract interface for interacting with various Optical Character Recognition (OCR) providers. Concrete implementations handle provider-specific API calls. + + +**Related Classes/Methods**: + +- `pipelex.cogt.ocr.ocr_worker_abstract` + + +### LLMDeck +Manages and validates configurations for various LLMs, ensuring correct setup and parameters for AI interactions. + + +**Related Classes/Methods**: + +- `pipelex.cogt.llm.llm_models.llm_deck` + + +### LLMModelLibrary +Provides access to loaded LLM model definitions and acts as a central repository for LLM models, enabling workers to retrieve necessary model information. + + +**Related Classes/Methods**: + +- `pipelex.cogt.llm.llm_models.llm_model_library` + + +### AI Configuration +Encapsulates specific configuration models (`LLMConfig`, `ImggConfig`, `OcrConfig`, `InferenceManagerConfig`) used to set up and run various AI tasks across the subsystem. + + +**Related Classes/Methods**: + +- `pipelex.cogt.config_cogt` + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/AI_ML_Integration_Extensibility.md b/docs/pages/advanced-customization/AI_ML_Integration_Extensibility.md new file mode 100644 index 000000000..feb559ace --- /dev/null +++ b/docs/pages/advanced-customization/AI_ML_Integration_Extensibility.md @@ -0,0 +1,47 @@ +```mermaid +graph LR + InferenceManager["InferenceManager"] + ContentGenerator["ContentGenerator"] + PluginManager["PluginManager"] + InferenceManager -- "leverages to discover and load" --> PluginManager + InferenceManager -- "provides AI model workers to" --> ContentGenerator + ContentGenerator -- "relies on to obtain and utilize" --> InferenceManager + PluginManager -- "enables extensibility for by providing" --> InferenceManager +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/GeneratedOnBoardings)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/demo)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +This subsystem provides a unified facade for interacting with various AI/ML models (LLMs, Image Generators, OCR engines) from different platforms and manages the integration and lifecycle of external AI/ML SDKs and plugins. It is a critical part of the `pipelex` framework, embodying the "Extensible AI Integration Layer" and supporting the "AI/ML Workflow Orchestration Framework/Library" and "Plugin/Extension Architecture" patterns. + +### InferenceManager +Acts as a unified facade and orchestrator for various AI/ML model workers, including Large Language Models (LLMs), Image Generators (Imgg), and Optical Character Recognition (OCR) engines. It is responsible for managing the lifecycle (setup, retrieval, and teardown) of these workers and provides a centralized point of access to different AI capabilities. It also supports dynamic registration of external LLM workers, reinforcing its role in extensibility. This component embodies aspects of the "Core Workflow Engine" (specifically the LLM Integration Layer) and the "Facade Pattern." + + +**Related Classes/Methods**: + +- `InferenceManager`:23-186 + + +### ContentGenerator +Specializes in generating diverse content (text, structured objects, images, OCR extractions, Jinja2 templated text) by leveraging the underlying AI/ML models. It encapsulates the logic for preparing prompts, handling model settings, and processing the outputs from various AI models. This component represents a specialized "Pipe Operator/Implementation" within the "LLM Integration Layer" and extends to other AI modalities like image generation and OCR. + + +**Related Classes/Methods**: + +- `ContentGenerator`:39-281 + + +### PluginManager +Manages the lifecycle of plugins, including their discovery, loading, registration, and provision to other parts of the system. It is crucial for enabling the extensibility of the `pipelex` framework, allowing new AI models, data sources, or custom logic to be integrated seamlessly. This component directly implements the "Extensibility & Plugin System" and "Plugin Management" architectural patterns. + + +**Related Classes/Methods**: + +- `PluginManager`:9-27 + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/AI_Service_Abstraction.md b/docs/pages/advanced-customization/AI_Service_Abstraction.md new file mode 100644 index 000000000..e28f05f70 --- /dev/null +++ b/docs/pages/advanced-customization/AI_Service_Abstraction.md @@ -0,0 +1,86 @@ +```mermaid +graph LR + InferenceManager["InferenceManager"] + LLMWorkerAbstract["LLMWorkerAbstract"] + ImggEngine["ImggEngine"] + OCRWorkerAbstract["OCRWorkerAbstract"] + ContentGenerator["ContentGenerator"] + PluginManager["PluginManager"] + Unclassified["Unclassified"] + InferenceManager -- "delegates tasks to" --> LLMWorkerAbstract + InferenceManager -- "delegates tasks to" --> ImggEngine + InferenceManager -- "delegates tasks to" --> OCRWorkerAbstract + InferenceManager -- "queries" --> PluginManager + ContentGenerator -- "uses" --> InferenceManager + PluginManager -- "provides implementations to" --> InferenceManager +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The core of the `pipelex` AI subsystem revolves around the `InferenceManager`, which acts as a central orchestrator for various AI tasks. It dynamically manages and delegates requests to specialized worker components like `LLMWorkerAbstract` for language models, `ImggEngine` for image generation, and `OCRWorkerAbstract` for optical character recognition. The `PluginManager` plays a crucial role in extending the system's capabilities by providing concrete implementations of these abstract workers to the `InferenceManager`. Higher-level components like `ContentGenerator` leverage the `InferenceManager` to compose complex AI workflows, demonstrating a clear separation of concerns and promoting extensibility through its plugin-based architecture. This design allows for flexible integration of diverse AI models and services, making the system adaptable to evolving AI landscapes. + +### InferenceManager +Acts as the central orchestrator and facade for all AI inference tasks. It provides a unified entry point for AI requests and intelligently routes them to the appropriate specialized AI worker or engine based on the task type (LLM, image generation, OCR). This component is fundamental for abstracting the underlying AI services and centralizing inference control, aligning with the "Workflow Orchestrator/Engine" pattern. + + +**Related Classes/Methods**: + +- `pipelex.cogt.inference.inference_manager.InferenceManager`:21-159 + + +### LLMWorkerAbstract +An abstract base class that defines the standardized interface for interacting with Large Language Models. It embodies the 'LLM Adapters/Providers' pattern, ensuring that different LLM backends can be seamlessly integrated and interchanged. This abstraction is critical for maintaining modularity and extensibility within the AI service layer. + + +**Related Classes/Methods**: + +- `pipelex.cogt.llm.llm_worker_abstract.LLMWorkerAbstract`:14-121 + + +### ImggEngine +A concrete component responsible for handling image generation requests. It encapsulates the specific logic and API calls required to interact with image generation services. This component represents a specific 'Tool/Service Integration' within the AI abstraction, demonstrating how diverse AI capabilities are integrated. + + +**Related Classes/Methods**: + +- `pipelex.cogt.imgg.imgg_engine.ImggEngine`:6-12 + + +### OCRWorkerAbstract +An abstract base class for Optical Character Recognition services, providing a unified interface for different OCR providers. Similar to `LLMWorkerAbstract`, it defines the contract for OCR services, allowing for interchangeable OCR backends and supporting the "Tool/Service Integrations" pattern. + + +**Related Classes/Methods**: + +- `pipelex.cogt.ocr.ocr_worker_abstract.OcrWorkerAbstract`:15-73 + + +### ContentGenerator +This component orchestrates multiple AI services (primarily LLMs) to generate more complex content. It acts as a higher-level consumer of the AI abstraction layer, composing and orchestrating calls to various AI services to produce structured or creative content. It represents a key aspect of the "Pipe Executors/Runners" by defining how AI tasks are combined into meaningful workflows. + + +**Related Classes/Methods**: + +- `pipelex.cogt.content_generation.content_generator.ContentGenerator`:38-280 + + +### PluginManager +Essential for the 'Extensibility/Plugin-based Architecture,' this component manages the discovery, loading, and provision of concrete AI service implementations (e.g., specific LLM or OCR providers). It facilitates the dynamic extension of AI service capabilities, which is crucial for an AI Workflow Orchestration Library to support new and evolving AI models and services. + + +**Related Classes/Methods**: + +- `pipelex.plugins.plugin_manager.PluginManager`:4-12 + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/AI_Service_Integration_Layer.md b/docs/pages/advanced-customization/AI_Service_Integration_Layer.md new file mode 100644 index 000000000..629298633 --- /dev/null +++ b/docs/pages/advanced-customization/AI_Service_Integration_Layer.md @@ -0,0 +1,95 @@ +```mermaid +graph LR + InferenceManager["InferenceManager"] + LLMDeck["LLMDeck"] + LLMModelProviderAbstract["LLMModelProviderAbstract"] + ImggWorkerAbstract["ImggWorkerAbstract"] + OcrWorkerAbstract["OcrWorkerAbstract"] + LLMPrompt["LLMPrompt"] + Plugins["Plugins"] + Unclassified["Unclassified"] + InferenceManager -- "orchestrates" --> LLMModelProviderAbstract + InferenceManager -- "orchestrates" --> ImggWorkerAbstract + InferenceManager -- "orchestrates" --> OcrWorkerAbstract + InferenceManager -- "utilizes" --> LLMDeck + LLMModelProviderAbstract -- "uses" --> LLMPrompt +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The AI Service Integration Layer serves as a crucial abstraction layer, centralizing the management and interaction with diverse AI services. The `InferenceManager` acts as the orchestrator, directing requests to specialized abstract provider interfaces (`LLMModelProviderAbstract`, `ImggWorkerAbstract`, `OcrWorkerAbstract`). The `LLMDeck` supports LLM operations by managing model configurations and selection. `LLMPrompt` is essential for dynamically constructing prompts for LLM interactions. The `Plugins` component ensures the extensibility of this layer, allowing for seamless integration of new AI service providers, thereby enhancing the overall flexibility and capabilities of the system. + +### InferenceManager +Acts as the primary facade for all AI inference operations within the subsystem. It orchestrates incoming requests, routes them to the appropriate specialized AI service providers (LLM, Image Generation, OCR), and manages their setup and configuration. It receives inference requests and delegates to specific `LLMModelProviderAbstract`, `ImggWorkerAbstract`, and `OcrWorkerAbstract` implementations, utilizing `LLMDeck` for LLM model selection and configuration. + + +**Related Classes/Methods**: + +- `pipelex.cogt.inference.inference_manager.InferenceManager`:23-186 + + +### LLMDeck +Manages a comprehensive registry of available LLM models, their configurations, and associated metadata (e.g., capabilities, cost parameters). It provides validation and selection logic for LLM settings to ensure compatibility and optimal usage. It is consulted by `InferenceManager` or `LLMModelProviderAbstract` implementations for model selection, configuration, and potentially cost tracking. + + +**Related Classes/Methods**: + +- `pipelex.cogt.llm.llm_models.llm_deck.LLMDeck`:19-161 + + +### LLMModelProviderAbstract +Defines the abstract interface for encapsulating specific API calls, authentication, and data formatting required to interact with individual Large Language Model providers (e.g., Anthropic, Google, Mistral, AWS Bedrock). It translates generic inference requests into provider-specific formats. It is called by `InferenceManager` and communicates directly with external LLM APIs, potentially utilizing `LLMPrompt` for dynamic prompt generation. + + +**Related Classes/Methods**: + +- `pipelex.cogt.llm.llm_models.llm_model_provider_abstract.LLMModelProviderAbstract`:8-42 + + +### ImggWorkerAbstract +Defines the abstract interface for handling provider-specific API interactions for various image generation services. It is called by `InferenceManager` and communicates with external Image Generation APIs. + + +**Related Classes/Methods**: + +- `pipelex.cogt.imgg.imgg_worker_abstract.ImggWorkerAbstract`:15-107 + + +### OcrWorkerAbstract +Defines the abstract interface for managing provider-specific API calls for Optical Character Recognition services. It is called by `InferenceManager` and communicates with external OCR APIs. + + +**Related Classes/Methods**: + +- `pipelex.cogt.ocr.ocr_worker_abstract.OcrWorkerAbstract`:15-71 + + +### LLMPrompt +Represents and manages the structure and content of prompts for LLM models. It facilitates the dynamic construction and formatting of prompts suitable for different LLM models and providers, potentially incorporating templating, context injection, and few-shot examples. It is used by `LLMModelProviderAbstract` implementations (and potentially `InferenceManager`) to prepare input for LLM calls. + + +**Related Classes/Methods**: + +- `pipelex.cogt.llm.llm_prompt.LLMPrompt`:13-85 + + +### Plugins +Provides extension points for integrating new or custom AI service providers into the AI Service Integration Layer, adhering to a plugin architecture. This allows for flexible expansion of supported AI services. + + +**Related Classes/Methods**: + +- `pipelex.plugins.__init__.py` + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/AI_Services_Integration.md b/docs/pages/advanced-customization/AI_Services_Integration.md new file mode 100644 index 000000000..8e393f771 --- /dev/null +++ b/docs/pages/advanced-customization/AI_Services_Integration.md @@ -0,0 +1,42 @@ +```mermaid +graph LR + Cognitive_Tools_AI_Operations["Cognitive Tools / AI Operations"] + AI_Model_Adapters_Integrations["AI Model Adapters / Integrations"] + Unclassified["Unclassified"] + Cognitive_Tools_AI_Operations -- "delegates to" --> AI_Model_Adapters_Integrations + AI_Model_Adapters_Integrations -- "receives requests from" --> Cognitive_Tools_AI_Operations +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The `pipelex` project leverages a modular architecture to integrate and manage various AI capabilities. The `Cognitive Tools / AI Operations` component, primarily embodied by the `InferenceManager`, acts as the central orchestrator for all AI-related tasks. It provides a high-level interface for requesting services from Large Language Models (LLMs), Optical Character Recognition (OCR), and Image Generation models. This component is responsible for routing these requests to the appropriate AI workers and managing their lifecycle. The `AI Model Adapters / Integrations` component, managed by the `PluginManager`, houses the concrete implementations for interacting with diverse third-party AI service providers. These adapters abstract away the complexities of each provider's API, enabling the `InferenceManager` to interact with different AI models seamlessly without needing to know their underlying specifics. This clear separation of concerns allows for easy extension and integration of new AI models and providers. + +### Cognitive Tools / AI Operations +This component provides a unified, abstract interface for interacting with various AI models (LLMs, OCR, Image Generation) and manages the routing and inference operations. It acts as a high-level orchestrator for AI capabilities within the pipeline, abstracting the underlying AI service providers. It serves as the primary entry point for the rest of the system to request AI services without needing to know the specifics of the underlying models or providers. + + +**Related Classes/Methods**: + +- `pipelex.cogt.inference.inference_manager.InferenceManager`:18-149 + + +### AI Model Adapters / Integrations +This component contains concrete implementations for integrating with specific third-party AI service providers (e.g., Anthropic, Google, Mistral, AWS Bedrock, OpenAI, FAL). It abstracts away the API specifics, authentication, and data formatting requirements of each external AI service, translating generic AI requests into provider-specific calls. + + +**Related Classes/Methods**: + +- `pipelex.plugins.plugin_manager.PluginManager`:4-12 + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/CLI_User_Interface.md b/docs/pages/advanced-customization/CLI_User_Interface.md new file mode 100644 index 000000000..072758b2e --- /dev/null +++ b/docs/pages/advanced-customization/CLI_User_Interface.md @@ -0,0 +1,81 @@ +```mermaid +graph LR + PipelexCLI["PipelexCLI"] + CLI_Commands["CLI Commands"] + Pipelex_Core["Pipelex Core"] + Pipelex_System_Configuration["Pipelex System Configuration"] + Pipelex_Pipeline["Pipelex Pipeline"] + Unclassified["Unclassified"] + PipelexCLI -- "delegates to" --> CLI_Commands + PipelexCLI -- "interacts with" --> Pipelex_System_Configuration + CLI_Commands -- "initiates operations on" --> Pipelex_Core + CLI_Commands -- "interacts with" --> Pipelex_System_Configuration + Pipelex_Core -- "manages" --> Pipelex_Pipeline + Pipelex_Core -- "interacts with" --> Pipelex_System_Configuration +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The Pipelex system's command-line interface is orchestrated by the PipelexCLI component, which acts as the central dispatcher for user commands. It delegates specific tasks to CLI Commands, such as run, build, and validate. These commands, in turn, interact with the Pipelex Core, which embodies the fundamental logic for pipeline execution, validation, and bundle management. The Pipelex Core relies on Pipelex System Configuration for managing settings and data, and directly manages Pipelex Pipeline instances. The Pipelex Builder component is responsible for generating pipelines based on user requirements, a function initiated by relevant CLI Commands. This architecture ensures a clear separation of concerns, with the CLI handling user interaction and command dispatch, while the core components manage the underlying pipeline logic and data. + +### PipelexCLI +Serves as the primary entry point for the command-line interface. It initializes the CLI application, registers available commands, parses user input, and dispatches control to the appropriate command handler. It acts as the orchestrator for user interactions with the Pipelex system. + + +**Related Classes/Methods**: + +- `pipelex.cli._cli.PipelexCLI`:14-27 + + +### CLI Commands +Each module within `pipelex.cli.commands` implements the specific business logic for a particular CLI command (e.g., run, build, validate). This includes handling command-line arguments, validating inputs, and invoking the relevant core Pipelex functionalities to perform tasks such as running, building, validating, or managing pipelines and concepts. + + +**Related Classes/Methods**: + +- `pipelex.cli.commands.run_cmd.run_cmd`:19-182 +- `pipelex.cli.commands.build_cmd.build_app` +- `pipelex.cli.commands.validate_cmd.validate_cmd`:33-148 + + +### Pipelex Core +Encapsulates the central functionalities of the Pipelex system, including pipeline execution, validation logic, and interaction with bundles. It provides the underlying mechanisms for processing and managing pipelines and their associated data. + + +**Related Classes/Methods**: + +- `pipelex.core.Pipelex`:64-292 +- `pipelex.core.interpreter.PipelexInterpreter`:17-92 + + +### Pipelex System Configuration +Manages the configuration settings and input/output data for the Pipelex system. This includes loading and saving JSON files for inputs and outputs, and potentially other system-wide settings. + + +**Related Classes/Methods**: + +- `pipelex.utils.io.load_json_dict_from_path`:106-128 +- `pipelex.utils.io.save_as_json_to_path`:59-81 + + +### Pipelex Pipeline +Represents the actual pipelines within the Pipelex system, including their definition, structure, and execution flow. This component is responsible for the logical representation and operational aspects of a pipeline. + + +**Related Classes/Methods**: + +- `pipelex.core.pipeline.execute_pipeline`:22-107 +- `pipelex.core.pipeline.get_required_pipe`:41-49 + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Client_API_Layer.md b/docs/pages/advanced-customization/Client_API_Layer.md new file mode 100644 index 000000000..b7fcaaa8d --- /dev/null +++ b/docs/pages/advanced-customization/Client_API_Layer.md @@ -0,0 +1,47 @@ +```mermaid +graph LR + Client_API_Layer["Client & API Layer"] + CLI_Command_Line_Interface_["CLI (Command-Line Interface)"] + Pipelex_Core_Engine_API_["Pipelex (Core Engine API)"] + CLI_Command_Line_Interface_ -- "interacts with" --> Pipelex_Core_Engine_API_ + Pipelex_Core_Engine_API_ -- "exposes API to" --> Client_API_Layer + click Client_API_Layer href "https://github.com/Pipelex/pipelex/blob/main/.codeboarding/Client_API_Layer.md" "Details" +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/GeneratedOnBoardings)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/demo)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The `Client & API Layer` is fundamentally important as it defines how users and other systems access the workflow orchestration capabilities. Separating it into `CLI` and `Pipelex (Core Engine API)` components aligns with the "Workflow Orchestration Engine / AI Workflow Framework" pattern by distinguishing between direct human interaction (CLI) and programmatic integration (API). This separation promotes modularity, allowing different types of clients to interact with the core engine through appropriate interfaces, which is crucial for extensibility and maintainability in an AI workflow framework. + +### Client & API Layer [[Expand]](./Client_API_Layer.md) +This layer serves as the primary interface for external users and systems to interact with the Pipelex engine. It encompasses both the command-line interface for direct user interaction and the programmatic API exposed by the core `Pipelex` engine for integration with other systems or custom scripts. It acts as the gateway for initiating, managing, and monitoring AI workflows. + + +**Related Classes/Methods**: + +- `pipelex.cli._cli` +- `pipelex.pipelex.Pipelex` (60:304) + + +### CLI (Command-Line Interface) +The `CLI` component provides a user-friendly command-line interface, enabling users to directly execute Pipelex commands, validate pipeline configurations, and manage workflow executions. It is the primary entry point for manual user interaction with the Pipelex system. + + +**Related Classes/Methods**: + +- `pipelex.cli._cli` + + +### Pipelex (Core Engine API) +While `Pipelex` represents the core orchestration engine, its public methods and attributes form the primary programmatic API. This API allows external clients, including the `CLI` and other integrated systems, to programmatically control and manage workflows, define pipelines, and interact with the engine's functionalities. + + +**Related Classes/Methods**: + +- `pipelex.pipelex.Pipelex` (60:304) + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Configuration_Extensibility_Management.md b/docs/pages/advanced-customization/Configuration_Extensibility_Management.md new file mode 100644 index 000000000..613ccce42 --- /dev/null +++ b/docs/pages/advanced-customization/Configuration_Extensibility_Management.md @@ -0,0 +1,63 @@ +```mermaid +graph LR + ConfigManager["ConfigManager"] + PipelexConfig["PipelexConfig"] + LibraryManager["LibraryManager"] + PluginManager["PluginManager"] + ConfigManager -- "populates" --> PipelexConfig + ConfigManager -- "provides configuration to" --> LibraryManager + ConfigManager -- "provides configuration to" --> PluginManager + PipelexConfig -- "is populated by" --> ConfigManager + PipelexConfig -- "provides data to" --> LibraryManager + PipelexConfig -- "provides data to" --> PluginManager + LibraryManager -- "depends on" --> ConfigManager + LibraryManager -- "utilizes" --> PipelexConfig + PluginManager -- "depends on" --> ConfigManager + PluginManager -- "utilizes" --> PipelexConfig +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/GeneratedOnBoardings)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/demo)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +This subsystem is responsible for managing all aspects of system configuration, workflow definitions, reusable components, and external plugins, ensuring the system's adaptability and future-proofing. + +### ConfigManager +Serves as the primary interface for loading, merging, and providing access to all system-wide and workflow-specific configuration settings. It handles the parsing of TOML-based definitions and ensures configuration consistency across the Pipelex ecosystem. + + +**Related Classes/Methods**: + +- `pipelex.tools.config.manager.ConfigManager` (22:222) + + +### PipelexConfig +Acts as the central data structure holding the loaded and validated configuration parameters for the entire Pipelex system. It provides a structured and accessible representation of all settings, including paths to libraries and plugin directories, enabling consistent access to configuration data. + + +**Related Classes/Methods**: + +- `pipelex.config.PipelexConfig` (113:116) + + +### LibraryManager +Orchestrates the discovery, loading, and validation of all workflow-related components (domains, concepts, pipes) defined in TOML files and registered Python classes. It acts as the central registry for reusable workflow building blocks, ensuring their availability and proper integration into the core engine. + + +**Related Classes/Methods**: + +- `pipelex.libraries.library_manager.Library_Manager` + + +### PluginManager +Manages the lifecycle of external plugins, including their discovery, loading, initialization, and teardown. This component is crucial for extending Pipelex's capabilities with new LLM providers, custom operators, or other integrations without modifying the core codebase, adhering to the microkernel/core-plugin pattern. + + +**Related Classes/Methods**: + +- `pipelex.plugins.plugin_manager.PluginManager` (8:26) + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Configuration_Secrets_Management.md b/docs/pages/advanced-customization/Configuration_Secrets_Management.md new file mode 100644 index 000000000..3e940ea7f --- /dev/null +++ b/docs/pages/advanced-customization/Configuration_Secrets_Management.md @@ -0,0 +1,54 @@ +```mermaid +graph LR + ConfigManager["ConfigManager"] + SecretsProviderAbstract["SecretsProviderAbstract"] + EnvSecretsProvider["EnvSecretsProvider"] + Unclassified["Unclassified"] + ConfigManager -- "uses" --> SecretsProviderAbstract + ConfigManager -- "depends on" --> SecretsProviderAbstract + SecretsProviderAbstract -- "is implemented by" --> EnvSecretsProvider + EnvSecretsProvider -- "implements" --> SecretsProviderAbstract +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The `pipelex` configuration and secrets management subsystem is built around a central `ConfigManager` that orchestrates the loading and merging of various configuration sources. This manager relies on an abstract `SecretsProviderAbstract` interface to securely retrieve sensitive information, promoting a clear separation of concerns. The `EnvSecretsProvider` is a concrete implementation of this interface, enabling the system to fetch secrets directly from environment variables. This design ensures flexibility in how configurations are managed and secrets are accessed, allowing for easy extension with different secret storage mechanisms while maintaining a consistent configuration loading process. + +### ConfigManager +The core component responsible for loading, parsing, validating, and providing access to all project-wide configuration settings. It acts as the central authority for configuration data, ensuring consistency and proper structure for workflow definitions and tool integrations. + + +**Related Classes/Methods**: + +- `pipelex.tools.config.manager:ConfigManager` + + +### SecretsProviderAbstract +Defines a standardized, abstract interface for retrieving sensitive information (e.g., API keys, credentials). This abstraction allows for multiple concrete implementations, promoting extensibility and decoupling the configuration manager from specific secret storage mechanisms. + + +**Related Classes/Methods**: + +- `pipelex.tools.secrets.secrets_provider_abstract:SecretsProviderAbstract` + + +### EnvSecretsProvider +A concrete implementation of the SecretsProviderAbstract interface, specifically designed to retrieve sensitive information from environment variables. This provides a common and easily deployable method for injecting secrets into the workflow execution environment. + + +**Related Classes/Methods**: + +- `pipelex.tools.secrets.env_secrets_provider:EnvSecretsProvider` + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Core_Orchestration_Engine.md b/docs/pages/advanced-customization/Core_Orchestration_Engine.md new file mode 100644 index 000000000..6849e08f0 --- /dev/null +++ b/docs/pages/advanced-customization/Core_Orchestration_Engine.md @@ -0,0 +1,61 @@ +```mermaid +graph LR + Pipelex_Core["Pipelex Core"] + Configuration_Manager["Configuration Manager"] + Pipeline_Manager["Pipeline Manager"] + Pipe_Router["Pipe Router"] + Pipelex_Core -- "initializes and coordinates" --> Configuration_Manager + Pipelex_Core -- "initializes and coordinates" --> Pipeline_Manager + Configuration_Manager -- "provides parsed pipeline definitions to" --> Pipelex_Core + Configuration_Manager -- "provides parsed pipeline definitions to" --> Pipeline_Manager + Pipeline_Manager -- "delegates individual pipe execution to" --> Pipe_Router + Pipeline_Manager -- "receives pipeline definitions from" --> Configuration_Manager + Pipe_Router -- "receives execution requests from" --> Pipeline_Manager + Pipe_Router -- "is initialized and coordinated by" --> Pipelex_Core +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/GeneratedOnBoardings)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/demo)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The `Core Orchestration Engine` subsystem is the heart of Pipelex, responsible for interpreting declarative pipeline definitions and managing the execution flow of individual operations. It embodies the core pipeline architecture pattern, ensuring that workflows are parsed, managed, and executed efficiently. + +### Pipelex Core +The primary entry point and top-level orchestrator of the entire Pipelex system. It is responsible for the overall system lifecycle, including initializing and coordinating the main managers (Configuration, Pipeline, and Pipe Router), acting as the central hub that initiates the orchestration process. + + +**Related Classes/Methods**: + +- `pipelex.pipelex.Pipelex` (60:304) + + +### Configuration Manager +Dedicated to loading, parsing, and validating declarative pipeline definitions from TOML files. It ensures that workflows are correctly structured and provides the necessary configuration data to other core components for execution. + + +**Related Classes/Methods**: + +- `pipelex.config` + + +### Pipeline Manager +Manages the lifecycle of defined pipelines, including their registration, retrieval, and overall execution flow. It oversees the high-level orchestration of a complete pipeline, delegating the execution of individual pipes to the Pipe Router. + + +**Related Classes/Methods**: + +- `pipelex.pipeline.pipeline_manager.PipelineManager` (13:43) + + +### Pipe Router +The core execution engine for individual 'pipes'. It dispatches execution requests for each pipe, manages data flow between them, and invokes the specific logic of each pipe operator. This component directly orchestrates the execution of individual steps within a pipeline. + + +**Related Classes/Methods**: + +- `pipelex.pipe_works.pipe_router.PipeRouter` (15:58) + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Core_Pipeline_Orchestrator.md b/docs/pages/advanced-customization/Core_Pipeline_Orchestrator.md new file mode 100644 index 000000000..dd7ea6d51 --- /dev/null +++ b/docs/pages/advanced-customization/Core_Pipeline_Orchestrator.md @@ -0,0 +1,72 @@ +```mermaid +graph LR + Pipelex_Application["Pipelex Application"] + Pipelex_Hub["Pipelex Hub"] + Pipeline_Manager["Pipeline Manager"] + Pipe_Router["Pipe Router"] + Pipe_Controllers["Pipe Controllers"] + Pipelex_Application -- "requests services from" --> Pipelex_Hub + Pipelex_Hub -- "provides services to" --> Pipelex_Application + Pipelex_Hub -- "grants access to" --> Pipeline_Manager + Pipeline_Manager -- "makes definitions available via" --> Pipelex_Hub + Pipelex_Hub -- "provides services to" --> Pipe_Router + Pipe_Router -- "registers with" --> Pipelex_Hub + Pipe_Router -- "directs requests to" --> Pipe_Controllers + Pipe_Controllers -- "executes logic dispatched by" --> Pipe_Router + Pipeline_Manager -- "provides definitions for" --> Pipe_Router +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The Core Pipeline Orchestrator subsystem is responsible for managing and executing AI workflows within the Pipelex system. It encompasses key modules like the Pipelex Application for initialization, the Pipelex Hub for service location, the Pipeline Manager for handling pipeline definitions, the Pipe Router for execution dispatch, and Pipe Controllers for concrete pipe execution logic. Its main purpose is to provide a robust and extensible framework for orchestrating AI pipelines. + +### Pipelex Application +Serves as the primary entry point for the Pipelex system. It is responsible for initializing the entire Pipelex environment, including core libraries and configurations, and initiating the overall orchestration process. This component embodies the "Engine" aspect of an AI Workflow Orchestration Library. + + +**Related Classes/Methods**: + +- `pipelex.pipelex`:61-305 + + +### Pipelex Hub +Functions as the central service locator and dependency injection point within the Pipelex ecosystem. It provides other components, such as the `Pipelex Application` and `Pipe Router`, with access to various managers and providers (e.g., `Pipeline Manager`), ensuring a decoupled and extensible architecture. + + +**Related Classes/Methods**: + +- `pipelex.hub` + + +### Pipeline Manager +Manages the lifecycle and declarative definitions of pipelines. Its core responsibility is to add new pipeline definitions and retrieve existing ones, focusing on the structural and configuration aspects of pipelines as defined via PLX. This component is crucial for the "Configuration Manager" and "DSL Parser/Interpreter" aspects. + + +**Related Classes/Methods**: + +- `pipelex.pipeline.pipeline_manager` + + +### Pipe Router +Acts as the dispatcher for pipe execution within a pipeline. It receives requests to execute specific pipes and directs these requests to the appropriate `Pipe Controllers` based on the pipe's type or configuration, ensuring correct control flow. This component is a key part of the "Pipeline Orchestrator/Engine." + + +**Related Classes/Methods**: + +- `pipelex.pipe_works.pipe_router` + + +### Pipe Controllers +Encapsulates the concrete execution logic and control flow for individual pipe steps. These are the specialized executors for different types of pipes (e.g., Condition, Parallel, Sequence, Batch), directly implementing the "Pipe Implementations" aspect of the architecture. + + +**Related Classes/Methods**: + +- `pipelex.pipe_controllers` + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Core_Workflow_Engine.md b/docs/pages/advanced-customization/Core_Workflow_Engine.md new file mode 100644 index 000000000..81252aec6 --- /dev/null +++ b/docs/pages/advanced-customization/Core_Workflow_Engine.md @@ -0,0 +1,46 @@ +```mermaid +graph LR + pipelex_pipelex_Pipelex["pipelex.pipelex.Pipelex"] + pipelex_hub_PipelexHub["pipelex.hub.PipelexHub"] + pipelex_pipeline_execute_execute_pipeline["pipelex.pipeline.execute.execute_pipeline"] + pipelex_pipelex_Pipelex -- "uses" --> pipelex_hub_PipelexHub + pipelex_pipelex_Pipelex -- "hands off to" --> pipelex_pipeline_execute_execute_pipeline + pipelex_pipeline_execute_execute_pipeline -- "queries" --> pipelex_hub_PipelexHub +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/GeneratedOnBoardings)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/demo)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The Core Workflow Engine subsystem is the central orchestrator of Pipelex applications, managing the entire lifecycle from definition parsing to execution. It embodies the Pipeline Architecture and leverages the Facade Pattern to provide a streamlined user experience. + +### pipelex.pipelex.Pipelex +This class serves as the Facade for the Pipelex framework, providing the high-level API for users to define, load, and run pipelines. It acts as the primary entry point, abstracting the complexities of pipeline parsing, component resolution, and execution initiation. It orchestrates the initial steps of a workflow, delegating to other components for specific tasks. + + +**Related Classes/Methods**: + +- `pipelex.pipelex.Pipelex`:61-301 + + +### pipelex.hub.PipelexHub +This class functions as a central Registry and Factory for managing and providing access to various pipeline components, configurations, and pre-defined pipeline structures. It is a key enabler of the Plugin/Extension Architecture and Strategy Pattern, allowing the framework to be extensible with new "pipe" types and operational units. It acts as a "hub" for available "pipes" that can be assembled into workflows. + + +**Related Classes/Methods**: + +- `pipelex.hub.PipelexHub`:39-299 + + +### pipelex.pipeline.execute.execute_pipeline +This function is the core Executor responsible for interpreting and running a given pipeline definition. It processes the sequence of operations defined in a pipeline, ensuring each "pipe" or step is executed correctly, and manages the data flow between them. This aligns directly with the Pipeline Architecture and potentially the Interpreter Pattern if it's parsing a DSL. + + +**Related Classes/Methods**: + +- `pipelex.pipeline.execute.execute_pipeline`:25-111 + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/DSL_Pipeline_Execution.md b/docs/pages/advanced-customization/DSL_Pipeline_Execution.md new file mode 100644 index 000000000..e431aeb6f --- /dev/null +++ b/docs/pages/advanced-customization/DSL_Pipeline_Execution.md @@ -0,0 +1,110 @@ +```mermaid +graph LR + PipeController["PipeController"] + PipeSequence["PipeSequence"] + PipeBatch["PipeBatch"] + PipeCondition["PipeCondition"] + PipeOperator["PipeOperator"] + Domain["Domain"] + Concept["Concept"] + LibraryManager["LibraryManager"] + PipeController -- "orchestrates" --> PipeSequence + PipeController -- "orchestrates" --> PipeBatch + PipeController -- "orchestrates" --> PipeCondition + PipeController -- "interprets DSL from" --> Domain + PipeController -- "interprets DSL from" --> Concept + PipeSequence -- "utilizes" --> PipeOperator + PipeBatch -- "utilizes" --> PipeOperator + PipeCondition -- "utilizes" --> PipeOperator + PipeCondition -- "controls flow to" --> PipeSequence + PipeCondition -- "controls flow to" --> PipeBatch + PipeOperator -- "retrieves functionalities via" --> LibraryManager + Domain -- "provides model for" --> PipeController + Domain -- "composed of" --> Concept + Concept -- "provides model for" --> PipeController + Concept -- "used within" --> Domain + LibraryManager -- "provides access to" --> PipeOperator + PipeController -- "interacts with" --> LibraryManager +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/GeneratedOnBoardings)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/demo)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The `pipelex` subsystem facilitates the creation and execution of domain-specific language (DSL) pipelines. The `PipeController` is the central component, responsible for interpreting the DSL defined by `Domain` and `Concept` elements and orchestrating the pipeline's execution. It leverages `PipeSequence` for sequential operations, `PipeBatch` for parallel processing, and `PipeCondition` for conditional branching, all of which rely on `PipeOperator` for atomic task execution. The `LibraryManager` is vital for extending the system's capabilities by dynamically providing access to various `PipeOperator` implementations, allowing for integration with external tools and services. This modular design ensures flexibility and extensibility in defining and executing complex workflows. + +### PipeController +The primary orchestrator and interpreter of the DSL. It takes the defined pipeline (composed of `Domain` and `Concept` elements) and manages its execution flow, delegating to specific control flow components. + + +**Related Classes/Methods**: + +- `PipeController`:14-79 + + +### PipeSequence +Implements sequential execution of `PipeOperator` instances. It ensures that pipeline steps are processed in a predefined, ordered manner, managing data flow between them. + + +**Related Classes/Methods**: + +- `PipeSequence`:23-195 + + +### PipeBatch +Manages the parallel or batch execution of multiple `PipeOperator` instances. This enables concurrent processing of data or tasks, optimizing workflow performance. + + +**Related Classes/Methods**: + +- `PipeBatch`:27-247 + + +### PipeCondition +Provides conditional branching logic within a pipeline. It evaluates specified criteria and directs the execution flow to different `PipeSequence` or `PipeBatch` paths based on the outcome. + + +**Related Classes/Methods**: + +- `PipeCondition`:32-379 + + +### PipeOperator +Serves as the abstract interface or base class for all individual, atomic operations within a pipeline. Concrete implementations perform specific tasks, such as LLM calls, data transformations, or external tool invocations. + + +**Related Classes/Methods**: + +- `PipeOperator`:16-90 + + +### Domain +Defines the high-level structure and entities of the domain-specific language. It provides the conceptual framework for organizing `Concept`s and constructing pipelines, establishing the overall workflow blueprint. + + +**Related Classes/Methods**: + +- `Domain`:405-406 + + +### Concept +Represents fundamental building blocks or atomic units within the `pipelex` DSL. These can be data types, abstract operations, or specific elements that `Domain` utilizes to define the granular aspects of workflows. + + +**Related Classes/Methods**: + +- `Concept`:448-449 + + +### LibraryManager +Manages the loading, registration, and access to external libraries, plugins, or integrations (e.g., LLM providers, specialized tools). It is crucial for the extensibility of the framework, allowing new `PipeOperator` implementations to be dynamically integrated. + + +**Related Classes/Methods**: + +- `LibraryManager`:56-406 + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Data_Context_Management.md b/docs/pages/advanced-customization/Data_Context_Management.md new file mode 100644 index 000000000..847946ca0 --- /dev/null +++ b/docs/pages/advanced-customization/Data_Context_Management.md @@ -0,0 +1,49 @@ +```mermaid +graph LR + WorkingMemory["WorkingMemory"] + Stuff["Stuff"] + StuffContent["StuffContent"] + WorkingMemory -- "manages" --> Stuff + WorkingMemory -- "accesses" --> StuffContent + Stuff -- "encapsulates" --> StuffContent + Stuff -- "is managed by" --> WorkingMemory + StuffContent -- "is encapsulated by" --> Stuff + StuffContent -- "is accessed by" --> WorkingMemory +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/GeneratedOnBoardings)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/demo)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +This subsystem is a critical part of the `pipelex` framework, providing the foundational mechanisms for managing data and execution context throughout an AI/ML workflow. It acts as the central nervous system for data flow, ensuring that information is consistently available and correctly interpreted across different pipeline stages. + +### WorkingMemory +The central, dynamic, in-memory store for all data (`Stuff` objects) and the execution context of a `pipelex` workflow. It provides a comprehensive API for managing this data, including adding, retrieving, updating, and removing `Stuff` objects, and handling their naming and aliasing. It ensures data consistency and facilitates data retrieval in various formats. This component is fundamental for state management and data passing between pipeline steps in an AI/ML workflow. + + +**Related Classes/Methods**: + +- `pipelex.core.working_memory.WorkingMemory`:39-376 + + +### Stuff +Represents the fundamental, atomic unit of data within the `pipelex` framework. It encapsulates a specific piece of content (`StuffContent`) along with essential metadata (e.g., `stuff_code`, `stuff_name`, `concept_code`). It provides methods for identifying the content type and safely accessing or converting the content into various formats. This component is crucial for enabling seamless data passing and state management in a DSL-driven workflow. + + +**Related Classes/Methods**: + +- `pipelex.core.stuff.Stuff`:26-198 + + +### StuffContent +Defines the actual data payloads that `Stuff` objects encapsulate. Each concrete `StuffContent` subclass (e.g., `TextContent`, `ImageContent`, `ListContent`, `PDFContent`) represents a specific type of data that can be processed within the `pipelex` framework. This supports the extensibility and plugin architecture by allowing new data types to be easily integrated into the workflow. + + +**Related Classes/Methods**: + +- `pipelex.core.stuff.StuffContent`:1-1000 + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Data_Knowledge_Management.md b/docs/pages/advanced-customization/Data_Knowledge_Management.md new file mode 100644 index 000000000..4de747c2f --- /dev/null +++ b/docs/pages/advanced-customization/Data_Knowledge_Management.md @@ -0,0 +1,88 @@ +```mermaid +graph LR + Data_Knowledge_Management["Data & Knowledge Management"] + Query_Processor["Query Processor"] + Language_Model_Interface["Language Model Interface"] + Tool_Executor["Tool Executor"] + Response_Formatter["Response Formatter"] + Unclassified["Unclassified"] + Query_Processor -- "sends parsed queries to" --> Language_Model_Interface + Query_Processor -- "may store context in or retrieve concepts from" --> Data_Knowledge_Management + Language_Model_Interface -- "receives queries from" --> Query_Processor + Language_Model_Interface -- "accesses and updates working memory and knowledge in" --> Data_Knowledge_Management + Language_Model_Interface -- "directs execution of tools via" --> Tool_Executor + Language_Model_Interface -- "sends generated responses to" --> Response_Formatter + Tool_Executor -- "receives execution directives from" --> Language_Model_Interface + Tool_Executor -- "stores results in or updates working memory in" --> Data_Knowledge_Management + Tool_Executor -- "returns execution results to" --> Language_Model_Interface + Response_Formatter -- "receives generated responses from" --> Language_Model_Interface + Response_Formatter -- "retrieves final state from" --> Data_Knowledge_Management + Data_Knowledge_Management -- "provides and receives data with" --> Query_Processor + Data_Knowledge_Management -- "provides and receives data with" --> Language_Model_Interface + Data_Knowledge_Management -- "provides and receives data with" --> Tool_Executor + Data_Knowledge_Management -- "provides and receives data with" --> Response_Formatter + click Data_Knowledge_Management href "https://github.com/Pipelex/pipelex/blob/main/.codeboarding/Data_Knowledge_Management.md" "Details" +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The `pipelex` system, at its core, orchestrates intelligent content generation and processing. The `Query Processor` initiates workflows by interpreting incoming requests. These requests are then passed to the `Language Model Interface`, which leverages the `Data & Knowledge Management` component to select and configure appropriate models. The `Language Model Interface` can also direct the `Tool Executor` to perform specialized tasks, integrating external capabilities. Finally, the `Response Formatter` structures the generated outputs for consumption. The `Data & Knowledge Management` component acts as a central repository for model configurations, routing logic, and inference backends, ensuring that the right models are used for the right tasks. + +### Data & Knowledge Management [[Expand]](./Data_Knowledge_Management.md) +Manages the configuration and specifications of various models, routing profiles, and inference backends. It is responsible for loading model definitions, building the model deck, and providing access to inference models and their associated backends. This component underpins the system's ability to select and utilize appropriate models for different tasks. + + +**Related Classes/Methods**: + +- `pipelex.cogt.models.model_manager.ModelManager`:20-149 +- `pipelex.cogt.llm.llm_job.LLMJob`:12-43 + + +### Query Processor +Handles incoming user queries or external triggers, including parsing and initial validation, preparing them for further processing by the AI workflow. This component would likely involve interpreting the request and setting up the necessary job parameters for the inference process. + + +**Related Classes/Methods**: + +- `pipelex.cogt.llm.llm_prompt.LLMPrompt`:11-85 + + +### Language Model Interface +Provides an abstraction layer for interacting with various large language models, responsible for understanding, reasoning, and generating responses or actions based on processed inputs and leveraging the system's knowledge. This component would manage the execution of LLM jobs. + + +**Related Classes/Methods**: + +- `pipelex.cogt.llm.llm_worker_abstract.LLMWorkerAbstract`:18-123 +- `pipelex.cogt.llm.llm_job.LLMJob.llm_job_before_start` + + +### Tool Executor +Manages the execution of external tools, APIs, or specialized functions as directed by the Language Model Interface, integrating external capabilities and real-world interactions into the AI workflow. This would involve interacting with various content generation modules. + + +**Related Classes/Methods**: + +- `pipelex.cogt.content_generation.content_generator.ContentGenerator`:37-278 + + +### Response Formatter +Structures and formats the final output from the AI workflow into a user-friendly or system-consumable representation, ensuring clarity and consistency. This would involve processing the reports generated by the LLM jobs. + + +**Related Classes/Methods**: + +- `pipelex.cogt.llm.llm_report.LLMReport` + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Data_Working_Memory.md b/docs/pages/advanced-customization/Data_Working_Memory.md new file mode 100644 index 000000000..b0fb86cc1 --- /dev/null +++ b/docs/pages/advanced-customization/Data_Working_Memory.md @@ -0,0 +1,81 @@ +```mermaid +graph LR + WorkingMemory["WorkingMemory"] + WorkingMemoryFactory["WorkingMemoryFactory"] + Stuff["Stuff"] + StuffContent["StuffContent"] + StuffFactory["StuffFactory"] + StuffContentFactory["StuffContentFactory"] + WorkingMemory -- "manages" --> Stuff + WorkingMemoryFactory -- "creates" --> WorkingMemory + WorkingMemoryFactory -- "uses" --> StuffFactory + Stuff -- "contains" --> StuffContent + Stuff -- "stored in" --> WorkingMemory + StuffContentFactory -- "creates" --> StuffContent + StuffFactory -- "creates" --> Stuff + StuffFactory -- "delegates to" --> StuffContentFactory +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/GeneratedOnBoardings)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/demo)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +Abstract Components Overview + +### WorkingMemory +Manages the dynamic state of the pipeline by storing and providing access to `Stuff` objects. It ensures data consistency and enables pipes to share and modify data throughout the workflow execution. + + +**Related Classes/Methods**: + +- `pipelex.core.working_memory` + + +### WorkingMemoryFactory +Initializes `WorkingMemory` instances, potentially populating them with initial `Stuff` objects. It acts as a controlled entry point for setting up the pipeline's state. + + +**Related Classes/Methods**: + +- `pipelex.core.working_memory_factory` + + +### Stuff +Encapsulates a single, typed data unit. It provides a consistent interface for accessing and manipulating its content, abstracting away the underlying data type. This is the fundamental data carrier within the pipeline. + + +**Related Classes/Methods**: + +- `pipelex.core.stuff` + + +### StuffContent +Holds the raw data and defines type-specific behaviors (e.g., rendering to string, JSON serialization). Subclasses handle specific data formats (text, image, PDF, structured data), ensuring proper data handling based on its type. + + +**Related Classes/Methods**: + +- `pipelex.core.stuff_content` + + +### StuffFactory +Acts as a controlled entry point for creating `Stuff` objects, abstracting the complexity of `StuffContent` instantiation. It ensures that `Stuff` objects are correctly formed with their appropriate content. + + +**Related Classes/Methods**: + +- `pipelex.core.stuff_factory` + + +### StuffContentFactory +Determines and instantiates the correct `StuffContent` subclass based on input data or specified types. This component is crucial for dynamic type handling and ensuring data integrity. + + +**Related Classes/Methods**: + +- `pipelex.core.stuff_factory` + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Monitoring_Reporting.md b/docs/pages/advanced-customization/Monitoring_Reporting.md new file mode 100644 index 000000000..5d1fc5f30 --- /dev/null +++ b/docs/pages/advanced-customization/Monitoring_Reporting.md @@ -0,0 +1,42 @@ +```mermaid +graph LR + Content_Generator["Content Generator"] + LLM_Report["LLM Report"] + Unclassified["Unclassified"] + Content_Generator -- "initiates" --> LLM_Report + LLM_Report -- "summarizes" --> Content_Generator +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The `pipelex` system orchestrates content generation pipelines, primarily focusing on LLM, image generation, and OCR tasks. At its core, the system leverages a `Content Generator` to execute these pipelines, which in turn relies on various job components and workers for specific tasks. The `LLM Report` component is responsible for summarizing the outcomes of LLM-specific pipeline executions, providing insights into their performance and results. The overall architecture emphasizes modularity, allowing for flexible integration of different content generation technologies and robust reporting capabilities. + +### Content Generator +This central component is responsible for orchestrating and executing various content generation pipelines, including LLM, image generation, and OCR. It acts as the primary driver for initiating and managing the flow of tasks within the system. + + +**Related Classes/Methods**: + +- `pipelex.cogt.content_generation.content_generator.ContentGenerator`:37-278 + + +### LLM Report +This component is dedicated to processing and presenting the results of LLM-based content generation pipelines. It aggregates data related to LLM job execution and generates structured reports, providing insights into the LLM's performance and output. + + +**Related Classes/Methods**: + +- `pipelex.cogt.llm.llm_report.LLMReport` + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Observability_Reporting.md b/docs/pages/advanced-customization/Observability_Reporting.md new file mode 100644 index 000000000..e57a585fe --- /dev/null +++ b/docs/pages/advanced-customization/Observability_Reporting.md @@ -0,0 +1,90 @@ +```mermaid +graph LR + Pipeline_Tracker["Pipeline Tracker"] + Reporting_Manager["Reporting Manager"] + Logging_Core["Logging Core"] + Log_Dispatcher["Log Dispatcher"] + Cost_Registry["Cost Registry"] + Mermaid_Utilities["Mermaid Utilities"] + Exception_Handling["Exception Handling"] + Pipeline_Tracker -- "utilizes" --> Mermaid_Utilities + Pipeline_Tracker -- "leverages" --> Exception_Handling + Reporting_Manager -- "manages" --> Cost_Registry + Reporting_Manager -- "uses" --> Logging_Core + Reporting_Manager -- "leverages" --> Exception_Handling + Logging_Core -- "delegates to" --> Log_Dispatcher + Cost_Registry -- "provides data to" --> Reporting_Manager +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/GeneratedOnBoardings)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/demo)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +This subsystem is critical for providing insights into pipeline execution, performance, and costs, thereby aiding in debugging and monitoring of AI workflows. It adheres to the Workflow Orchestration Engine / AI Workflow Framework patterns by centralizing monitoring and reporting functionalities, which are essential for managing complex, long-running processes. + +### Pipeline Tracker +Responsible for monitoring and recording the execution flow, state, and key metrics of individual pipelines and their constituent pipes. It provides the raw data necessary for understanding pipeline behavior and identifying bottlenecks. + + +**Related Classes/Methods**: + +- `pipelex.reporting.tracker.PipelineTracker` (1:1) + + +### Reporting Manager +Aggregates and processes data collected from various sources (like the Pipeline Tracker and Cost Registry) to generate comprehensive reports on pipeline performance, costs, and overall operational health. It acts as the central hub for report generation. + + +**Related Classes/Methods**: + +- `pipelex.reporting.manager.ReportingManager` (1:1) + + +### Logging Core +Provides the foundational logging infrastructure for the entire system, centralizing the collection of events, warnings, and errors from various components. It ensures a consistent logging standard across the observability subsystem. + + +**Related Classes/Methods**: + +- `pipelex.logging.core.LoggingCore` (1:1) + + +### Log Dispatcher +Handles the routing and distribution of log messages received from the Logging Core to various output destinations, such as console, files, or external monitoring systems. It ensures logs are delivered efficiently to where they are needed. + + +**Related Classes/Methods**: + +- `pipelex.logging.dispatcher.LogDispatcher` (1:1) + + +### Cost Registry +Maintains a dedicated registry for tracking and registering costs associated with pipeline execution, particularly for resource-intensive operations like LLM API calls. This component is crucial for cost analysis and optimization. + + +**Related Classes/Methods**: + +- `pipelex.reporting.cost_registry.CostRegistry` (1:1) + + +### Mermaid Utilities +Offers utility functions specifically designed to generate Mermaid syntax, enabling the creation of visual diagrams (e.g., flowcharts) that represent pipeline structures and execution flows, aiding in visual debugging and understanding. + + +**Related Classes/Methods**: + +- `pipelex.reporting.mermaid.MermaidUtilities` (1:1) + + +### Exception Handling +Provides a centralized and consistent mechanism for catching, processing, and reporting exceptions across the system. Within the observability context, it ensures that errors are properly logged and can be integrated into reports for debugging and monitoring. + + +**Related Classes/Methods**: + +- `pipelex.common.exception_handling.ExceptionHandling` (1:1) + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/PLX_Definition_Registry.md b/docs/pages/advanced-customization/PLX_Definition_Registry.md new file mode 100644 index 000000000..4168248c0 --- /dev/null +++ b/docs/pages/advanced-customization/PLX_Definition_Registry.md @@ -0,0 +1,80 @@ +```mermaid +graph LR + pipelex_core_interpreter["pipelex.core.interpreter"] + pipelex_libraries_library_manager["pipelex.libraries.library_manager"] + pipelex_core_concepts_concept_library["pipelex.core.concepts.concept_library"] + pipelex_core_pipes_pipe_library["pipelex.core.pipes.pipe_library"] + pipelex_tools_config_manager["pipelex.tools.config.manager"] + pipelex_libraries_library_config["pipelex.libraries.library_config"] + pipelex_libraries_library_manager -- "relies on" --> pipelex_tools_config_manager + pipelex_libraries_library_manager -- "queries" --> pipelex_libraries_library_config + pipelex_libraries_library_manager -- "invokes" --> pipelex_core_interpreter + pipelex_libraries_library_manager -- "populates" --> pipelex_core_concepts_concept_library + pipelex_libraries_library_manager -- "populates" --> pipelex_core_pipes_pipe_library + pipelex_core_interpreter -- "implicitly interacts with" --> pipelex_core_concepts_concept_library + pipelex_core_interpreter -- "implicitly interacts with" --> pipelex_core_pipes_pipe_library +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +This subsystem is responsible for the core interpretation of the Pipelex Domain-Specific Language (PLX) for defining pipelines and concepts. It also acts as the central repository and manager for these definitions and any loaded external libraries, ensuring they are parsed, registered, and made available throughout the system. + +### pipelex.core.interpreter +The core engine for the Pipelex DSL. It parses raw PLX content (TOML-based) into structured internal blueprints (representing concepts, pipes, domains) and serializes these internal representations back into PLX strings. It is the primary interface for interacting with the PLX language itself. + + +**Related Classes/Methods**: + +- `pipelex.core.interpreter` + + +### pipelex.libraries.library_manager +Orchestrates the discovery, loading, and registration of all Pipelex definitions (concepts, pipes, domains) from various sources. It utilizes configuration to locate PLX files, delegates parsing to the `interpreter`, and then populates the respective concept, pipe, and domain registries. + + +**Related Classes/Methods**: + +- `pipelex.libraries.library_manager` + + +### pipelex.core.concepts.concept_library +Serves as the central, in-memory registry for all defined "concepts" (data types or knowledge representations) within Pipelex. It provides mechanisms to add new concepts and ensures their availability and consistency across the system. + + +**Related Classes/Methods**: + +- `pipelex.core.concepts.concept_library` + + +### pipelex.core.pipes.pipe_library +Manages the central, in-memory registry of all defined "pipes" (individual steps or complete pipelines) in Pipelex. It allows for the addition of new pipe definitions and makes them available for orchestration and execution. + + +**Related Classes/Methods**: + +- `pipelex.core.pipes.pipe_library` + + +### pipelex.tools.config.manager +Provides a centralized service for managing and accessing project-wide configuration settings. This includes locating root directories, project names, and general configuration parameters that influence how PLX definitions and libraries are found and loaded. + + +**Related Classes/Methods**: + +- `pipelex.tools.config.manager` + + +### pipelex.libraries.library_config +Specializes in managing paths and configurations specific to different types of Pipelex libraries and assets, such as LLM decks, templates, and plugins. It provides methods to retrieve these specific directory paths. + + +**Related Classes/Methods**: + +- `pipelex.libraries.library_config` + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Pipe_Execution_Units.md b/docs/pages/advanced-customization/Pipe_Execution_Units.md new file mode 100644 index 000000000..6732d7fa1 --- /dev/null +++ b/docs/pages/advanced-customization/Pipe_Execution_Units.md @@ -0,0 +1,94 @@ +```mermaid +graph LR + Pipe_Operators["Pipe Operators"] + Core_Memory_State_Management["Core Memory / State Management"] + AI_Model_Adapters_Integrations["AI Model Adapters / Integrations"] + Cognitive_Tools_AI_Operations["Cognitive Tools / AI Operations"] + Pipe_Controllers["Pipe Controllers"] + Utility_Tools["Utility Tools"] + Unclassified["Unclassified"] + Core_Memory_State_Management -- "provides input to" --> Pipe_Operators + Pipe_Operators -- "sends output to" --> Core_Memory_State_Management + Pipe_Operators -- "utilizes" --> AI_Model_Adapters_Integrations + Pipe_Operators -- "utilizes" --> Cognitive_Tools_AI_Operations + Pipe_Controllers -- "invokes" --> Pipe_Operators + Pipe_Operators -- "may use" --> Utility_Tools +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The `pipelex` project's core subsystem is designed around a flexible pipeline execution model. Pipe Operators define the atomic units of work, interacting with Core Memory / State Management for data. Pipe Controllers orchestrate execution, managing flow. AI Model Adapters / Integrations and Cognitive Tools / AI Operations provide specialized AI capabilities to Pipe Operators. Utility Tools offer general services like configuration. This architecture promotes modularity for easy extension and adaptation. + +### Pipe Operators +This component encapsulates the specific operational logic for individual steps (pipes) within a pipeline. It includes the core definitions and interfaces for pipes, as well as their concrete implementations, serving as the atomic units of work. Pipe Operators are responsible for executing tasks such as interacting with AI models, executing custom functions, or performing data transformations, adhering to a clear knowledge-in, knowledge-out contract. + + +**Related Classes/Methods**: + +- `pipelex.core.pipes.pipe_abstract.PipeAbstract`:16-94 +- `pipelex.pipe_operators.pipe_operator.PipeOperator`:19-102 +- `pipelex.pipe_operators.llm.pipe_llm.LLMPipe` + + +### Core Memory / State Management +Manages the state and data flow within and between pipeline steps, ensuring consistent access to information. This component is crucial for maintaining context and sharing results across different pipe operations. + + +**Related Classes/Methods**: + +- `pipelex.core.memory` + + +### AI Model Adapters / Integrations +Provides standardized interfaces and mechanisms for integrating with various external AI models and services. This component abstracts away the complexities of different model APIs, allowing Pipe Operators to interact with AI capabilities uniformly. + + +**Related Classes/Methods**: + +- `pipelex.cogt.model_backends` +- `pipelex.plugins` + + +### Cognitive Tools / AI Operations +Encapsulates higher-level AI functionalities and cognitive operations, such as Large Language Model (LLM) interactions, image generation, and Optical Character Recognition (OCR). These tools provide specialized AI capabilities that Pipe Operators can leverage. + + +**Related Classes/Methods**: + +- `pipelex.cogt.llm` +- `pipelex.cogt.img_gen` +- `pipelex.cogt.ocr` +- `pipelex.cogt.content_generation` + + +### Pipe Controllers +Orchestrates the execution of pipes, managing control flow logic such as sequencing, parallelism, and conditional execution. This component dictates the order and manner in which Pipe Operators are invoked. + + +**Related Classes/Methods**: + +- `pipelex.pipe_controllers` +- `pipelex.pipe_controllers.sequence.sequence_controller.SequenceController` + + +### Utility Tools +Provides a collection of general-purpose utilities and helper functions used across the system. This includes functionalities like configuration management, secret handling, and templating, supporting various operational needs of other components. + + +**Related Classes/Methods**: + +- `pipelex.tools` +- `pipelex.tools.config.config_manager.ConfigManager` + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Pipe_Operators.md b/docs/pages/advanced-customization/Pipe_Operators.md new file mode 100644 index 000000000..950d2b9ec --- /dev/null +++ b/docs/pages/advanced-customization/Pipe_Operators.md @@ -0,0 +1,108 @@ +```mermaid +graph LR + Pipe_Operator_Base["Pipe Operator Base"] + Specialized_Pipe_Operators["Specialized Pipe Operators"] + Pipe_Abstract["Pipe Abstract"] + Core_Engine["Core Engine"] + LLM_Adapters_Providers["LLM Adapters/Providers"] + Input_Output_Management["Input/Output Management"] + Pipe_Operator_Factories["Pipe Operator Factories"] + Configuration_Layer["Configuration Layer"] + Pipe_Operator_Base -- "inherits from" --> Pipe_Abstract + Pipe_Operator_Base -- "interacts with" --> Input_Output_Management + Specialized_Pipe_Operators -- "inherits from" --> Pipe_Operator_Base + Specialized_Pipe_Operators -- "utilizes" --> LLM_Adapters_Providers + Pipe_Abstract -- "extended by" --> Pipe_Operator_Base + Core_Engine -- "executes" --> Pipe_Operator_Base + Core_Engine -- "receives results from" --> Pipe_Operator_Base + LLM_Adapters_Providers -- "provides services to" --> Specialized_Pipe_Operators + Input_Output_Management -- "manages data for" --> Pipe_Operator_Base + Pipe_Operator_Factories -- "creates instances of" --> Specialized_Pipe_Operators + Configuration_Layer -- "provides settings to" --> Pipe_Operator_Base +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/GeneratedOnBoardings)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/demo)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +Abstract Components Overview of the Pipelex system. + +### Pipe Operator Base +The abstract base class (pipelex.pipe_operators.pipe_operator) that defines the common interface and core execution logic for all specialized pipe operators. It ensures a standardized contract for how individual operations integrate into a pipeline. + + +**Related Classes/Methods**: + +- `pipelex.pipe_operators.pipe_operator` + + +### Specialized Pipe Operators +Concrete implementations of the Pipe Operator Base, each designed for a specific task. This includes pipelex.pipe_operators.pipe_llm (for LLM calls), pipelex.pipe_operators.pipe_img_gen (for image generation), pipelex.pipe_operators.pipe_ocr (for OCR), pipelex.pipe_operators.pipe_func (for custom Python functions), and pipelex.pipe_operators.pipe_jinja2 (for templating). + + +**Related Classes/Methods**: + +- `pipelex.pipe_operators.pipe_llm` +- `pipelex.pipe_operators.pipe_img_gen` +- `pipelex.pipe_operators.pipe_ocr` +- `pipelex.pipe_operators.pipe_func` +- `pipelex.pipe_operators.pipe_jinja2` + + +### Pipe Abstract +The most fundamental abstract base (pipelex.core.pipe_abstract.PipeAbstract) that establishes the absolute minimum interface required for any component to be considered a "pipe" within the Pipelex system. Pipe Operator Base extends this. + + +**Related Classes/Methods**: + +- `pipelex.core.pipe_abstract.PipeAbstract` (13:81) + + +### Core Engine +The central orchestration component responsible for parsing pipeline definitions, managing the execution flow, and invoking Pipe Operators in the correct sequence (sequential, parallel, conditional). + + +**Related Classes/Methods**: + +- `pipelex.core` + + +### LLM Adapters/Providers +A set of modules that abstract the complexities of interacting with various Large Language Model services (e.g., Anthropic, Google, Mistral AI). They provide a consistent API for PipeLLM to consume. + + +**Related Classes/Methods**: + +- `pipelex.llms` + + +### Input/Output Management +Handles the standardized flow of data into and out of individual Pipe Operators, ensuring data consistency, type validation, and proper transfer between pipeline stages. + + +**Related Classes/Methods**: + +- `pipelex.io` + + +### Pipe Operator Factories +Responsible for dynamically creating instances of Specialized Pipe Operators at runtime, typically based on declarative workflow definitions (e.g., TOML files) and associated blueprints. + + +**Related Classes/Methods**: + +- `pipelex.factories` + + +### Configuration Layer +Manages and provides system-wide and pipeline-specific configuration settings, which influence the behavior and parameters of Pipe Operators during their instantiation and execution. + + +**Related Classes/Methods**: + +- `pipelex.config` + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Pipe_Operators_Working_Memory.md b/docs/pages/advanced-customization/Pipe_Operators_Working_Memory.md new file mode 100644 index 000000000..0a540991c --- /dev/null +++ b/docs/pages/advanced-customization/Pipe_Operators_Working_Memory.md @@ -0,0 +1,98 @@ +```mermaid +graph LR + Pipe_Operator_Abstract_Base_["Pipe Operator (Abstract Base)"] + LLM_Operator["LLM Operator"] + Working_Memory["Working Memory"] + Stuff_Data_Unit_["Stuff (Data Unit)"] + Stuff_Factory["Stuff Factory"] + Jinja2_Renderer["Jinja2 Renderer"] + Template_Library["Template Library"] + Pipe_Operator_Abstract_Base_ -- "interacts with" --> Working_Memory + LLM_Operator -- "inherits from" --> Pipe_Operator_Abstract_Base_ + LLM_Operator -- "interacts with" --> Working_Memory + LLM_Operator -- "utilizes" --> Jinja2_Renderer + LLM_Operator -- "utilizes" --> Template_Library + Stuff_Factory -- "adds stuff to" --> Working_Memory + Stuff_Factory -- "creates" --> Stuff_Data_Unit_ + Stuff_Data_Unit_ -- "is stored in" --> Working_Memory + Template_Library -- "provides templates to" --> Jinja2_Renderer + Working_Memory -- "provides context to" --> Jinja2_Renderer + click Working_Memory href "https://github.com/Pipelex/pipelex/blob/main/.codeboarding/Working_Memory.md" "Details" +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/GeneratedOnBoardings)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/demo)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +This subsystem is crucial for the execution layer of the `pipelex` project, embodying the "Execution Layer" and "Data/Knowledge Management" architectural patterns. The central components are chosen for their fundamental role in defining, executing, and managing data flow within the pipeline. + +### Pipe Operator (Abstract Base) +Defines the abstract interface and common execution flow (`run_pipe`, `_dry_run_operator_pipe`, `_run_operator_pipe`) for all concrete pipe operators. It acts as the base for specialized data processing units. + + +**Related Classes/Methods**: + +- `pipelex.pipe_operators.pipe_operator` +- `pipelex.pipe_operators.pipe_operator:run_pipe` +- `pipelex.pipe_operators.pipe_operator:_dry_run_operator_pipe` +- `pipelex.pipe_operators.pipe_operator:_run_operator_pipe` + + +### LLM Operator +A concrete implementation of `pipe_operator` specialized for interacting with Large Language Models (LLMs). It handles input validation, content generation (`_llm_gen_object_stuff_content`), and structuring LLM outputs. + + +**Related Classes/Methods**: + +- `pipelex.pipe_operators.llm.pipe_llm` +- `pipelex.pipe_operators.llm.pipe_llm:_llm_gen_object_stuff_content` + + +### Working Memory [[Expand]](./Working_Memory.md) +Serves as the central, transient data store for `stuff` objects during pipeline execution. Provides methods for adding, retrieving, and managing data. + + +**Related Classes/Methods**: + +- `pipelex.core.memory.working_memory` + + +### Stuff (Data Unit) +Represents the atomic unit of data within the system. It encapsulates content and metadata, providing methods to access content in various formats. + + +**Related Classes/Methods**: + +- `pipelex.core.stuffs.stuff` + + +### Stuff Factory +Facilitates the creation and combination of `stuff` objects from diverse inputs (strings, concepts, blueprints). It acts as the primary mechanism for populating `working_memory` with initial data or new data generated by operators. + + +**Related Classes/Methods**: + +- `pipelex.core.stuffs.stuff_factory` + + +### Jinja2 Renderer +Handles the actual rendering of Jinja2 templates by integrating context (often derived from `working_memory`) into the templating environment. + + +**Related Classes/Methods**: + +- `pipelex.tools.templating.jinja2_rendering` + + +### Template Library +Manages a collection of templates, loading them from configuration files and providing access to them for rendering. + + +**Related Classes/Methods**: + +- `pipelex.tools.templating.template_library` + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Pipelex_Client_CLI_SDK_.md b/docs/pages/advanced-customization/Pipelex_Client_CLI_SDK_.md new file mode 100644 index 000000000..bcc0a85cf --- /dev/null +++ b/docs/pages/advanced-customization/Pipelex_Client_CLI_SDK_.md @@ -0,0 +1,23 @@ +```mermaid +graph LR + ComponentA["ComponentA"] + ComponentA -- "uses" --> ComponentB +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +This is a placeholder description. + +### ComponentA +Description of ComponentA + + +**Related Classes/Methods**: + + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Pipelex_Core_User_Interface.md b/docs/pages/advanced-customization/Pipelex_Core_User_Interface.md new file mode 100644 index 000000000..7a503fe93 --- /dev/null +++ b/docs/pages/advanced-customization/Pipelex_Core_User_Interface.md @@ -0,0 +1,60 @@ +```mermaid +graph LR + Core_Initializer["Core Initializer"] + Service_Hub["Service Hub"] + CLI_Executor["CLI Executor"] + API_Client["API Client"] + Core_Initializer -- "invokes configuration setup on" --> Service_Hub + Core_Initializer -- "registers core components with" --> Service_Hub + CLI_Executor -- "retrieves system components from" --> Service_Hub + CLI_Executor -- "may trigger actions on components via" --> Service_Hub + CLI_Executor -- "triggers initialization routines in" --> Core_Initializer + API_Client -- "requests pipeline execution components from" --> Service_Hub + Service_Hub -- "provides requested components to" --> API_Client +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/GeneratedOnBoardings)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/demo)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The `Pipelex Core & User Interface` subsystem serves as the primary interaction layer and foundational control mechanism for the Pipelex application. It encompasses the initial setup, central orchestration, and user-facing interfaces (CLI and programmatic API). + +### Core Initializer +Manages the initial setup, configuration loading, and bootstrapping of the Pipelex environment. It serves as the primary application entry point, preparing the system for operation and integrating necessary dependencies. This component is crucial for any Data Processing Library or AI Workflow Orchestration Tool as it establishes the runtime environment. + + +**Related Classes/Methods**: + +- `pipelex.pipelex`:61-305 + + +### Service Hub +Acts as the central control unit and dependency injector, providing a unified access point to all core components, configurations, and specialized managers/providers throughout the system. It orchestrates the overall system by enabling various parts to discover and interact without tight coupling, supporting the modular and plugin architecture. This aligns with the "Orchestration" and "Plugin/Extension Architecture" patterns. + + +**Related Classes/Methods**: + +- `pipelex.hub` + + +### CLI Executor +Processes command-line inputs, translates them into Pipelex operations, and provides user feedback. It handles administrative and operational tasks such as migration, initialization, validation, and information display, serving as the primary command-line interface for user interaction. This is a standard component for any robust Data Processing Library, providing direct user control. + + +**Related Classes/Methods**: + +- `pipelex.cli.commands.__init__` + + +### API Client +Enables external systems or applications to programmatically interact with Pipelex. It facilitates the initiation of pipeline executions and retrieval of results, providing a programmatic interface for integration into larger systems. Essential for an AI Workflow Orchestration Tool, allowing integration into broader MLOps ecosystems. + + +**Related Classes/Methods**: + +- `pipelex.client.client` + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Pipeline_Data_Store.md b/docs/pages/advanced-customization/Pipeline_Data_Store.md new file mode 100644 index 000000000..f3beb1efb --- /dev/null +++ b/docs/pages/advanced-customization/Pipeline_Data_Store.md @@ -0,0 +1,45 @@ +```mermaid +graph LR + Pipeline_Working_Memory["Pipeline Working Memory"] + Working_Memory_Factory["Working Memory Factory"] + Unclassified["Unclassified"] + Working_Memory_Factory -- "creates and manages" --> Pipeline_Working_Memory + Working_Memory_Factory -- "provides instances to" --> Pipeline_Orchestrator + Pipeline_Working_Memory -- "exchanges data with" --> Pipeline_Orchestrator + Pipeline_Working_Memory -- "provides input to and receives output from" --> Pipe_Executors + Pipeline_Working_Memory -- "stores and retrieves data for" --> Pipe_Controllers +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The memory subsystem in `pipelex.core.memory` is foundational for managing data within AI pipelines. It consists of two primary components: the `Pipeline Working Memory` and the `Working Memory Factory`. The `Working Memory Factory` is responsible for creating and initializing instances of the `Pipeline Working Memory`, which then serves as the central, ephemeral data store for the entire pipeline execution. This `Pipeline Working Memory` facilitates data exchange and state management by allowing components like the `Pipeline Orchestrator`, `Pipe Executors`, and `Pipe Controllers` to store, retrieve, and manipulate data artifacts throughout the pipeline's lifecycle. This clear separation of concerns ensures efficient data handling and flexible pipeline execution. + +### Pipeline Working Memory +This component, represented by the `pipelex.core.memory.WorkingMemory` class, serves as the central, ephemeral data store for a running pipeline. It is responsible for holding, managing, and providing access to all data artifacts ("stuffs") that flow between different stages of a pipeline. It ensures data continuity, facilitates data exchange, and maintains the overall state and context of the pipeline execution, which is crucial for stateful AI workflows. It offers methods for adding, retrieving, and manipulating these data artifacts, including support for aliases and various content types. + + +**Related Classes/Methods**: + +- `pipelex.core.memory.WorkingMemory`:37-295 + + +### Working Memory Factory +This component, implemented by the `pipelex.core.memory.WorkingMemoryFactory` class, is dedicated to the creation and management of `Pipeline Working Memory` instances. It provides various static methods to instantiate `WorkingMemory` objects from diverse sources, such as raw text, images, PDFs, lists of "stuffs," or compact memory representations. It abstracts the complex instantiation logic, ensuring that new, properly initialized working memory instances are available for each pipeline run, and can also generate mock memory for dry-run scenarios. + + +**Related Classes/Methods**: + +- `pipelex.core.memory.WorkingMemoryFactory`:20-248 + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Pipeline_Definition_Data_Schema.md b/docs/pages/advanced-customization/Pipeline_Definition_Data_Schema.md new file mode 100644 index 000000000..530a97f7a --- /dev/null +++ b/docs/pages/advanced-customization/Pipeline_Definition_Data_Schema.md @@ -0,0 +1,104 @@ +```mermaid +graph LR + pipelex_libraries_library_manager["pipelex.libraries.library_manager"] + pipelex_core_interpreter["pipelex.core.interpreter"] + pipelex_core_pipes_pipe_library["pipelex.core.pipes.pipe_library"] + pipelex_core_pipes_pipe_factory["pipelex.core.pipes.pipe_factory"] + pipelex_core_pipes_pipe_input_spec["pipelex.core.pipes.pipe_input_spec"] + pipelex_core_concepts_concept_library["pipelex.core.concepts.concept_library"] + pipelex_core_concepts_concept_blueprint["pipelex.core.concepts.concept_blueprint"] + pipelex_core_domains_domain_library["pipelex.core.domains.domain_library"] + pipelex_libraries_library_manager -- "invokes" --> pipelex_core_interpreter + pipelex_libraries_library_manager -- "populates" --> pipelex_core_pipes_pipe_library + pipelex_libraries_library_manager -- "populates" --> pipelex_core_concepts_concept_library + pipelex_libraries_library_manager -- "populates" --> pipelex_core_domains_domain_library + pipelex_core_interpreter -- "generates blueprints for" --> pipelex_core_pipes_pipe_factory + pipelex_core_interpreter -- "processes" --> pipelex_core_pipes_pipe_input_spec + pipelex_core_pipes_pipe_library -- "references" --> pipelex_core_pipes_pipe_input_spec + pipelex_core_pipes_pipe_factory -- "constructs objects for" --> pipelex_core_pipes_pipe_library + pipelex_core_pipes_pipe_input_spec -- "defines input schema for" --> pipelex_core_pipes_pipe_library + pipelex_core_concepts_concept_library -- "manages" --> pipelex_core_concepts_concept_blueprint + pipelex_core_domains_domain_library -- "organizes" --> pipelex_core_concepts_concept_library +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/GeneratedOnBoardings)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/demo)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The `Pipeline Definition & Data Schema` subsystem is responsible for managing the declarative definition of pipelines and the underlying data structures that define the data flowing through them. It translates external blueprints into executable internal representations. + +### pipelex.libraries.library_manager +Orchestrates the loading and registration of all pipeline-related definitions (pipes, concepts, domains) from external sources (e.g., TOML files) into their respective in-memory libraries. It acts as the primary entry point for external configuration. + + +**Related Classes/Methods**: + +- `pipelex.libraries.library_manager` + + +### pipelex.core.interpreter +Translates declarative TOML pipeline and concept blueprints into internal structured data representations. It is the core parser for the system's declarative configuration, converting human-readable definitions into a format the system can understand. + + +**Related Classes/Methods**: + +- `pipelex.core.interpreter` + + +### pipelex.core.pipes.pipe_library +Acts as a central registry for all available `Pipe` definitions, allowing for storage, retrieval, and validation of pipe blueprints. It ensures that all defined pipes are consistently managed and accessible. + + +**Related Classes/Methods**: + +- `pipelex.core.pipes.pipe_library` + + +### pipelex.core.pipes.pipe_factory +Constructs concrete `Pipe` objects from their blueprint definitions, ensuring correct instantiation based on the parsed declarative structure. It bridges the gap between a declarative definition and an executable object. + + +**Related Classes/Methods**: + +- `pipelex.core.pipes.pipe_factory` + + +### pipelex.core.pipes.pipe_input_spec +Defines the expected input schema for a pipe, including required concept codes and validation rules. It ensures data consistency and type safety within the pipeline by specifying what a pipe expects to receive. + + +**Related Classes/Methods**: + +- `pipelex.core.pipes.pipe_input_spec` + + +### pipelex.core.concepts.concept_library +Manages a collection of all defined data concepts, providing mechanisms for adding, retrieving, and validating them. It ensures a consistent understanding of data types and structures across the system. + + +**Related Classes/Methods**: + +- `pipelex.core.concepts.concept_library` + + +### pipelex.core.concepts.concept_blueprint +Defines the declarative structure and validation rules for a data concept, serving as the template for creating `Concept` objects. It is the actual schema definition for data concepts. + + +**Related Classes/Methods**: + +- `pipelex.core.concepts.concept_blueprint` + + +### pipelex.core.domains.domain_library +Manages a collection of defined data domains, which categorize and organize concepts, providing a hierarchical structure for data organization. This helps in structuring and managing a large number of concepts. + + +**Related Classes/Methods**: + +- `pipelex.core.domains.domain_library` + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Pipeline_Execution_Core.md b/docs/pages/advanced-customization/Pipeline_Execution_Core.md new file mode 100644 index 000000000..89f709512 --- /dev/null +++ b/docs/pages/advanced-customization/Pipeline_Execution_Core.md @@ -0,0 +1,112 @@ +```mermaid +graph LR + Pipelex_Environment_Bootstrapper["Pipelex Environment Bootstrapper"] + Pipelex_Hub["Pipelex Hub"] + Pipeline_Manager["Pipeline Manager"] + Pipe_Controllers["Pipe Controllers"] + Pipe_Operators["Pipe Operators"] + Working_Memory["Working Memory"] + Pipeline_Tracker["Pipeline Tracker"] + Inference_Manager["Inference Manager"] + Unclassified["Unclassified"] + Pipelex_Environment_Bootstrapper -- "initializes" --> Pipelex_Hub + Pipelex_Hub -- "provides access to" --> Pipeline_Manager + Pipelex_Hub -- "provides access to" --> Pipe_Controllers + Pipelex_Hub -- "provides access to" --> Working_Memory + Pipelex_Hub -- "provides access to" --> Pipeline_Tracker + Pipelex_Hub -- "provides access to" --> Inference_Manager + Pipe_Controllers -- "orchestrates execution by invoking" --> Pipe_Operators + Pipe_Controllers -- "reads/writes data from/to" --> Working_Memory + Pipe_Controllers -- "updates execution status in" --> Pipeline_Tracker + Pipe_Operators -- "processes data from" --> Working_Memory + Pipe_Operators -- "writes results to" --> Working_Memory + Pipe_Operators -- "may interact with" --> Inference_Manager +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The Pipelex system is initiated by the `Pipelex Environment Bootstrapper`, which sets up the foundational environment and the `Pipelex Hub`. The `Pipelex Hub` acts as a central service locator, providing core components like the `Pipeline Manager`, `Pipe Controllers`, `Working Memory`, `Pipeline Tracker`, and `Inference Manager` with necessary dependencies. The `Pipeline Manager` oversees the lifecycle of pipeline instances. `Pipe Controllers` are responsible for orchestrating the execution flow of pipelines, interpreting declarative definitions, and invoking `Pipe Operators`. `Pipe Operators` are the atomic units that perform specific AI/ML tasks or data transformations. During execution, `Pipe Controllers` and `Pipe Operators` interact with `Working Memory` for transient data storage and `Pipeline Tracker` for monitoring and recording the execution path. The `Inference Manager`, a specialized service for AI inference, is accessed through the `Pipelex Hub` and utilized by `Pipe Operators` for AI-specific tasks. + +### Pipelex Environment Bootstrapper +Initializes the entire Pipelex system, including the Pipelex Hub, setting up the foundational environment for pipeline execution. + + +**Related Classes/Methods**: + +- `pipelex.__init__` + + +### Pipelex Hub +Acts as a central service locator and dependency injection container. It provides other components within the execution core with access to various managers, providers, and workers (e.g., PipelineManager, InferenceManager). + + +**Related Classes/Methods**: + +- `pipelex.hub` + + +### Pipeline Manager +Manages the lifecycle of pipeline instances. It is responsible for creating new pipelines, retrieving existing ones by their run ID, and handling their state or persistence. + + +**Related Classes/Methods**: + +- `pipelex.pipeline.pipeline_manager` + + +### Pipe Controllers +The primary orchestrator of pipeline execution. It interprets declarative pipeline definitions (e.g., sequences, parallel branches, conditional logic, batch processing) and drives the execution flow by invoking appropriate Pipe Operators. + + +**Related Classes/Methods**: + +- `pipelex.pipe_controllers` + + +### Pipe Operators +Implements the core logic for individual "pipe" types. These are the atomic units of work within a pipeline, performing specific AI/ML tasks (e.g., LLM calls, OCR, image generation) or data transformations. + + +**Related Classes/Methods**: + +- `pipelex.pipe_operators` + + +### Working Memory +Serves as the transient data store for 'Stuff' objects that flow between pipes during a pipeline's execution. It manages the state and data context for the currently running pipeline, ensuring data consistency and availability for subsequent steps. + + +**Related Classes/Methods**: + +- `pipelex.core.memory.working_memory` + + +### Pipeline Tracker +Monitors and records the execution path of a pipeline, including the sequence of executed pipes, conditional branches taken, and parallel tasks. It is crucial for debugging, auditing, and generating visual representations of the execution flow. + + +**Related Classes/Methods**: + +- `pipelex.pipeline.track.pipeline_tracker` + + +### Inference Manager +A specialized service for managing and executing AI inference tasks, often interacting with external LLM providers. It is accessed via the Pipelex Hub and utilized by Pipe Operators for AI-specific tasks. + + +**Related Classes/Methods**: + +- `pipelex.cogt.inference.inference_manager` + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Pipeline_Observability_Reporting.md b/docs/pages/advanced-customization/Pipeline_Observability_Reporting.md new file mode 100644 index 000000000..538b0f7b6 --- /dev/null +++ b/docs/pages/advanced-customization/Pipeline_Observability_Reporting.md @@ -0,0 +1,66 @@ +```mermaid +graph LR + Pipeline_Tracker["Pipeline Tracker"] + Flowchart_Generator["Flowchart Generator"] + Activity_Manager["Activity Manager"] + Reporting_Manager["Reporting Manager"] + Usage_Registry["Usage Registry"] + Pipeline_Tracker -- "provides data to" --> Flowchart_Generator + Activity_Manager -- "provides details to" --> Pipeline_Tracker + Usage_Registry -- "provides data to" --> Reporting_Manager +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/GeneratedOnBoardings)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/demo)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The Pipeline Observability & Reporting subsystem encompasses core components responsible for monitoring, tracking, and reporting on pipeline execution and resource usage. + +### Pipeline Tracker +Manages the structural definition and runtime tracking of the pipeline. It records pipeline steps, their interconnections, and execution flow, forming a graph representation of the workflow. It also handles the setup and teardown of tracking sessions. This component is fundamental for understanding the dynamic behavior and structure of any executed pipeline. + + +**Related Classes/Methods**: + +- `pipelex.pipeline.track.pipeline_tracker` + + +### Flowchart Generator +Visualizes the pipeline structure. It takes tracked pipeline data from the Pipeline Tracker and generates Mermaid flowchart code, including subgraphs and properly tagged edges, to provide a clear graphical representation of the workflow for analysis and debugging. This is crucial for human-readable insights into complex workflows. + + +**Related Classes/Methods**: + +- `pipelex.pipeline.track.flow_chart` + + +### Activity Manager +Manages the lifecycle of individual pipeline activities. It handles the setup and teardown phases for activities, orchestrating how individual activity outputs and states are managed during pipeline execution. This component ensures that each step within the pipeline is properly observed and its state is available for tracking. + + +**Related Classes/Methods**: + +- `pipelex.pipeline.activity.activity_manager` + + +### Reporting Manager +Collects, aggregates, and generates comprehensive reports on pipeline usage metrics, with a particular focus on LLM (Large Language Model) job costs, such as token usage. This component is vital for operational insights, cost analysis, and performance monitoring in AI/ML workflows. + + +**Related Classes/Methods**: + +- `pipelex.reporting.reporting_manager` + + +### Usage Registry +Serves as a dedicated data store for recording and managing usage statistics, primarily token consumption for LLM inference jobs. It acts as the backend for the Reporting Manager, providing a persistent and queryable source of truth for resource utilization. + + +**Related Classes/Methods**: + +- `pipelex.reporting.reporting_manager.UsageRegistry`:20-27 + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Pipeline_Orchestrator.md b/docs/pages/advanced-customization/Pipeline_Orchestrator.md new file mode 100644 index 000000000..baec71769 --- /dev/null +++ b/docs/pages/advanced-customization/Pipeline_Orchestrator.md @@ -0,0 +1,42 @@ +```mermaid +graph LR + PipelineController["PipelineController"] + PipeOperator["PipeOperator"] + Unclassified["Unclassified"] + PipelineController -- "composes" --> PipeOperator + PipelineController -- "orchestrates" --> PipeOperator +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +This system provides a framework for orchestrating AI tasks. The PipelineController acts as the central execution engine, responsible for interpreting workflows and managing the execution flow of various tasks. It coordinates the invocation of individual PipeOperator instances, which represent atomic, self-contained steps within the pipeline, such as LLM calls or data transformations. The overall purpose is to ensure robust and repeatable execution of AI workflows. + +### PipelineController +This component serves as the core execution engine for the entire pipeline. It is responsible for interpreting the defined workflow, managing the execution flow (sequential, parallel, conditional, batch), and coordinating the invocation of individual `PipeOperator` instances. It embodies the "Workflow Orchestrator/Engine" pattern, ensuring robust and repeatable execution of AI tasks. + + +**Related Classes/Methods**: + +- `pipelex.pipelines.pipeline_controller`:10-25 + + +### PipeOperator +Represents an atomic, self-contained step or task within a pipeline. Each `PipeOperator` encapsulates a specific operation, such as calling an LLM, performing data transformation, or integrating with an external service. It aligns with the "Pipe Executors/Runners" pattern, providing the modular units of work that the `PipelineController` orchestrates. + + +**Related Classes/Methods**: + +- `pipelex.pipelines.pipe_operator`:16-90 + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Reporting_Monitoring.md b/docs/pages/advanced-customization/Reporting_Monitoring.md new file mode 100644 index 000000000..cddb66ac9 --- /dev/null +++ b/docs/pages/advanced-customization/Reporting_Monitoring.md @@ -0,0 +1,67 @@ +```mermaid +graph LR + ReportingManager["ReportingManager"] + PipelineTracker["PipelineTracker"] + FlowChartGenerator["FlowChartGenerator"] + ActivityManager["ActivityManager"] + ActivityHandler["ActivityHandler"] + PipelineTracker -- "provides structural execution data to" --> FlowChartGenerator + FlowChartGenerator -- "consumes structural data from" --> PipelineTracker + ActivityManager -- "orchestrates the operations of" --> ActivityHandler + ActivityHandler -- "is orchestrated by" --> ActivityManager +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The `Reporting & Monitoring` subsystem is critical for providing visibility into pipeline execution, resource consumption, and operational activities within the `pipelex` AI Workflow Orchestration Library. It encompasses modules responsible for tracking pipeline flow, logging detailed activities, and generating usage reports. + +### ReportingManager +Orchestrates the collection and aggregation of usage data, specifically focusing on token consumption from LLM inference. It is responsible for generating financial reports to manage AI workflow costs. It manages an internal `UsageRegistry`. + + +**Related Classes/Methods**: + +- `pipelex.reporting.reporting_manager.ReportingManager`:30-122 + + +### PipelineTracker +Monitors and records the execution path of a pipeline, capturing each step (e.g., pipes, batches, conditions, choices) to construct a logical flow representation. Its primary output is data used for visual flowcharts. + + +**Related Classes/Methods**: + +- `pipelex.pipeline.track.pipeline_tracker.PipelineTracker`:30-364 + + +### FlowChartGenerator +Specializes in generating Mermaid syntax for visual flowcharts based on the structural data provided by the `PipelineTracker`. + + +**Related Classes/Methods**: + +- `pipelex.pipeline.track.flow_chart.FlowChartGenerator` + + +### ActivityManager +Manages the overall lifecycle of activity tracking within pipelines, including the proper initialization and cleanup of activity-related resources. + + +**Related Classes/Methods**: + +- `pipelex.pipeline.activity.activity_manager.ActivityManager`:16-46 + + +### ActivityHandler +Processes and categorizes various types of content generated during pipeline execution (e.g., text, images, structured data, Mermaid diagrams), assigning unique identifiers to each activity. + + +**Related Classes/Methods**: + +- `pipelex.pipeline.activity.activity_handler.ActivityHandler`:23-132 + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Reporting_Observability.md b/docs/pages/advanced-customization/Reporting_Observability.md new file mode 100644 index 000000000..3c7b567c2 --- /dev/null +++ b/docs/pages/advanced-customization/Reporting_Observability.md @@ -0,0 +1,55 @@ +```mermaid +graph LR + Pipelex_Core_Execution_Components["Pipelex Core Execution Components"] + LocalObserver["LocalObserver"] + ReportingManager["ReportingManager"] + Unclassified["Unclassified"] + Pipelex_Core_Execution_Components -- "Exposes data to" --> LocalObserver + LocalObserver -- "Feeds data to" --> ReportingManager +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The Pipelex system's core execution is driven by the `Pipelex Core Execution Components`, which are responsible for building and running pipelines defined by `PipelexBundleSpec` objects. The `FlowFactory` constructs a high-level `Flow` representation, while `WorkingMemory` manages the runtime state. During this execution, the `LocalObserver` passively collects detailed event and metric data, storing it locally. This collected data is then consumed by the `ReportingManager`, which processes and aggregates it to generate comprehensive reports, particularly focusing on resource usage and costs associated with pipeline operations. This architecture ensures a clear separation of concerns between pipeline execution, data observation, and reporting. + +### Pipelex Core Execution Components +This logical grouping represents the operational parts of the Pipelex system responsible for building, interpreting, and executing pipelines. It includes the core logic for defining pipeline flows, managing pipe specifications, and handling working memory during execution. These components are the source of all observable events, metrics, and logs. + + +**Related Classes/Methods**: + +- `pipelex.builder.builder.assemble_pipelex_bundle_spec`:53-123 +- `pipelex.builder.flow.Flow`:31-54 +- `pipelex.builder.flow_factory.FlowFactory`:22-119 +- `pipelex.core.memory.working_memory.WorkingMemory`:42-385 + + +### LocalObserver +The `LocalObserver` acts as the primary data collection agent within the subsystem. It actively monitors and captures specific events, performance metrics, and resource usage directly from the `Pipelex Core Execution Components` during pipeline execution. It writes these observations to local JSONL files. + + +**Related Classes/Methods**: + +- `pipelex.observer.local_observer.LocalObserver`:10-35 + + +### ReportingManager +The `ReportingManager` is the central hub for processing, aggregating, and presenting all collected observability data. It takes raw or preprocessed data from the `LocalObserver`, transforms it into meaningful insights, and manages the output of reports, particularly for LLM token usage and costs. + + +**Related Classes/Methods**: + +- `pipelex.reporting.reporting_manager.ReportingManager`:29-123 + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Runtime_Data_State.md b/docs/pages/advanced-customization/Runtime_Data_State.md new file mode 100644 index 000000000..dad357bad --- /dev/null +++ b/docs/pages/advanced-customization/Runtime_Data_State.md @@ -0,0 +1,82 @@ +```mermaid +graph LR + WorkingMemory["WorkingMemory"] + Stuff["Stuff"] + StuffContent["StuffContent"] + StuffArtefact["StuffArtefact"] + WorkingMemoryFactory["WorkingMemoryFactory"] + StuffFactory["StuffFactory"] + WorkingMemory -- "manages" --> Stuff + WorkingMemory -- "delegates to" --> Stuff + Stuff -- "encapsulates" --> StuffContent + Stuff -- "delegates to" --> StuffContent + Stuff -- "creates" --> StuffArtefact + StuffArtefact -- "represents" --> StuffContent + WorkingMemoryFactory -- "creates" --> WorkingMemory + StuffFactory -- "creates" --> Stuff + StuffFactory -- "creates" --> StuffContent +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The `pipelex` core memory and data artifact subsystem is designed around the `WorkingMemory` component, which serves as the central, transient store for all data artifacts, represented by `Stuff` objects. `WorkingMemoryFactory` is responsible for initializing `WorkingMemory` instances. Each `Stuff` object encapsulates `StuffContent`, which holds the raw data and handles type-specific operations, including rendering. `StuffFactory` facilitates the creation of `Stuff` and `StuffContent` from various inputs. For rendering purposes, `Stuff` can create a `StuffArtefact` from its `StuffContent`, providing a specialized, renderable view suitable for templating engines. This architecture ensures a clear separation of concerns, with dedicated components for data storage, content encapsulation, rendering representation, and object creation. + +### WorkingMemory +The central, transient store for all data artifacts (`Stuff` objects) during pipeline execution. It manages the lifecycle of these artifacts, including adding, retrieving, updating, and aliasing, ensuring data availability and integrity across pipeline steps. + + +**Related Classes/Methods**: + +- `pipelex.core.memory.working_memory.WorkingMemory`:37-344 + + +### Stuff +Represents a single, unified data artifact within the pipeline, encapsulating `StuffContent`. It provides a consistent interface for accessing and validating content, abstracting underlying data types to fulfill the "knowledge-in, knowledge-out" contract of the pipeline. + + +**Related Classes/Methods**: + +- `pipelex.core.stuffs.stuff.Stuff`:26-194 + + +### StuffContent +Holds the raw data of a `Stuff` object and handles type-specific operations. It is responsible for rendering content into various output formats (e.g., HTML, JSON, plain text, Markdown) and supports content creation from basic types, performing the actual data validation and manipulation. + + +**Related Classes/Methods**: + +- `pipelex.core.stuffs.stuff_content.StuffContent`:30-70 + + +### StuffArtefact +Provides a specialized, renderable view of `StuffContent`, primarily focused on rendering tagged content for templating engines (e.g., Jinja2). It facilitates the extraction and presentation of content in a format suitable for dynamic generation within the pipeline. + + +**Related Classes/Methods**: + +- `pipelex.core.stuffs.stuff_artefact.StuffArtefact`:13-60 + + +### WorkingMemoryFactory +Manages the creation and initialization of `WorkingMemory` instances. It can construct `WorkingMemory` from various initial states, such as pre-existing data (text, image, PDF), compact memory representations, or for specific operational modes like dry runs. + + +**Related Classes/Methods**: + +- `pipelex.core.memory.working_memory_factory.WorkingMemoryFactory`:20-248 + + +### StuffFactory +Handles the creation of `Stuff` and `StuffContent` objects from diverse inputs, including raw strings, concept definitions, blueprints, or by combining existing `Stuff` instances. It also manages error conditions that may arise during the creation process. + + +**Related Classes/Methods**: + +- `pipelex.core.stuffs.stuff_factory.StuffFactory`:38-259 + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/System_Services_Interfaces.md b/docs/pages/advanced-customization/System_Services_Interfaces.md new file mode 100644 index 000000000..4f83932fc --- /dev/null +++ b/docs/pages/advanced-customization/System_Services_Interfaces.md @@ -0,0 +1,57 @@ +```mermaid +graph LR + Pipelex_Client["Pipelex Client"] + Templating_Engine["Templating Engine"] + Pipeline_Tracker["Pipeline Tracker"] + Reporting_Manager["Reporting Manager"] + Pipelex_Client -- "initiates monitoring of and queries data from" --> Pipeline_Tracker + Pipelex_Client -- "provides data to" --> Templating_Engine + Templating_Engine -- "renders content for" --> Pipelex_Client + Pipeline_Tracker -- "feeds execution data to" --> Reporting_Manager +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/GeneratedOnBoardings)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/demo)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The Pipelex system facilitates AI/ML workflow orchestration, with the `Pipelex Client` serving as the primary user interface for defining and initiating pipelines. This client interacts with a `Templating Engine` to dynamically generate content, such as LLM prompts or configurations, ensuring flexible workflow execution. As pipelines run, the `Pipeline Tracker` diligently monitors and records all execution events and statuses, acting as the central data collection point for observability. This raw execution data is then fed to the `Reporting Manager`, which aggregates and processes it to generate insightful reports on pipeline performance and errors, providing critical analytics for users. + +### Pipelex Client +Serves as the primary programmatic interface for users to define, configure, and initiate AI/ML workflows. It acts as a facade to the underlying orchestration engine, abstracting its complexity. This aligns with the "User Interfaces (with Programmatic API)" pattern, providing the main entry point for developers. + + +**Related Classes/Methods**: + +- `PipelexClient`:16-121 + + +### Templating Engine +Manages the rendering of dynamic content using Jinja2 templates. This is crucial for generating prompts for LLMs, dynamic configurations, or other variable data required during workflow execution, supporting the "Extensible AI Integration Layer" and "Core Workflow Engine" patterns by enabling dynamic content generation. + + +**Related Classes/Methods**: + +- `render_jinja2`:34-102 + + +### Pipeline Tracker +Monitors and records the execution flow, status, and key events of pipelines. It provides the foundational data for understanding pipeline performance and debugging, directly supporting "Monitoring, Logging, & Reporting (with Activity Tracking)". It's the core data collection point for observability. + + +**Related Classes/Methods**: + +- `PipelineTracker`:30-358 + + +### Reporting Manager +Aggregates and processes data collected by the `Pipeline Tracker` to generate comprehensive reports on pipeline performance, errors, and other metrics, providing actionable insights. This aligns with "Monitoring, Logging, & Reporting (with Reporting & Analytics)", transforming raw tracking data into meaningful information. + + +**Related Classes/Methods**: + +- `ReportingManager`:30-122 + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Templating_Engine.md b/docs/pages/advanced-customization/Templating_Engine.md new file mode 100644 index 000000000..d57478173 --- /dev/null +++ b/docs/pages/advanced-customization/Templating_Engine.md @@ -0,0 +1,64 @@ +```mermaid +graph LR + Templating_Pipe_Operator["Templating Pipe Operator"] + Template_Management_Service["Template Management Service"] + Jinja2_Rendering_Engine["Jinja2 Rendering Engine"] + PLX_Templating_Interpreter["PLX Templating Interpreter"] + Templating_Pipe_Operator -- "invokes" --> Jinja2_Rendering_Engine + PLX_Templating_Interpreter -- "defines structure for" --> Templating_Pipe_Operator + Template_Management_Service -- "provides templates to" --> Jinja2_Rendering_Engine + PLX_Templating_Interpreter -- "provides definitions to" --> Template_Management_Service + Jinja2_Rendering_Engine -- "requests templates from" --> Template_Management_Service +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The `Templating Engine` subsystem is primarily bounded by the `pipelex.tools.templating` package, which provides the core templating utilities, and `pipelex.pipe_operators.jinja2.pipe_jinja2`, which integrates Jinja2 into the pipeline execution. Additionally, components within `pipelex.core.interpreter` are crucial for how templating is defined and serialized within the Pipelex DSL. The main flow involves defining templates and templating operations via the PLX DSL, managing these templates, and then executing them within the pipeline to generate dynamic content. + +### Templating Pipe Operator +Acts as the primary interface for integrating Jinja2 templating into the `pipelex` pipeline execution flow. It receives templating instructions from the pipeline orchestrator and orchestrates the rendering process, enabling dynamic content generation within pipes. This component is the direct bridge between the pipeline and the templating functionality. + + +**Related Classes/Methods**: + +- `pipelex.pipe_operators.jinja2.pipe_jinja2` + + +### Template Management Service +Manages the lifecycle of templates, including loading, storing, validating, and providing access to them. It ensures templates are correctly defined and available for the rendering process, often sourced from PLX (TOML) definitions. This component acts as the central repository and gatekeeper for all templates. + + +**Related Classes/Methods**: + +- `pipelex.tools.templating.template_library` +- `pipelex.tools.templating.template_preprocessor` +- `pipelex.hub.get_template_provider`:329-330 + + +### Jinja2 Rendering Engine +Encapsulates the core Jinja2 functionality, responsible for creating and managing Jinja2 environments, executing the rendering process, and applying custom filters. It injects context (e.g., pipeline variables, previous pipe outputs) into templates to produce the final dynamic content. + + +**Related Classes/Methods**: + +- `pipelex.tools.templating.jinja2_rendering` +- `pipelex.tools.templating.jinja2_environment` +- `pipelex.tools.templating.jinja2_filters` +- `pipelex.core.stuffs.stuff_artefact:render_tagged_for_jinja2` + + +### PLX Templating Interpreter +Handles the conversion and serialization of Jinja2 pipe definitions into the Pipelex Language (PLX) string representation. This component is crucial for the DSL's ability to define, store, and interpret templating operations within pipelines, ensuring consistency with the overall `pipelex` DSL. + + +**Related Classes/Methods**: + +- `pipelex.core.interpreter.jinja2_pipe_to_plx_string`:398-418 +- `pipelex.core.interpreter.serialize_jinja2_pipe`:781-803 + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/User_Interaction_Layer.md b/docs/pages/advanced-customization/User_Interaction_Layer.md new file mode 100644 index 000000000..90fc8d303 --- /dev/null +++ b/docs/pages/advanced-customization/User_Interaction_Layer.md @@ -0,0 +1,67 @@ +```mermaid +graph LR + CLI_Interface["CLI Interface"] + Python_Client_SDK["Python Client/SDK"] + Pipeline_Tracker["Pipeline Tracker"] + Activity_Manager["Activity Manager"] + Unclassified["Unclassified"] + CLI_Interface -- "depends on" --> Python_Client_SDK + CLI_Interface -- "receives reports from" --> Pipeline_Tracker + Python_Client_SDK -- "updates" --> Pipeline_Tracker + Pipeline_Tracker -- "reports data to" --> Python_Client_SDK + Python_Client_SDK -- "utilizes" --> Activity_Manager + Pipeline_Tracker -- "provides data for reports to" --> CLI_Interface + Activity_Manager -- "reports activity details to" --> Pipeline_Tracker +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The Pipelex system provides both a command-line interface (CLI) and a Python SDK for managing data pipelines. The `CLI Interface` leverages the `Python Client/SDK` for core operations and receives status reports from the `Pipeline Tracker`. The `Python Client/SDK` is the primary programmatic interface, allowing users to define, update, and monitor pipelines. It interacts with the `Pipeline Tracker` to update pipeline definitions and receive execution data, and directly utilizes the `Activity Manager` for granular control over pipeline steps. The `Pipeline Tracker` is central to managing pipeline lifecycles, status, and historical data, receiving detailed activity reports from the `Activity Manager` and providing overall pipeline status to both the `Python Client/SDK` and the `CLI Interface`. The `Activity Manager` is responsible for overseeing individual pipeline steps, providing granular execution details to the `Pipeline Tracker`. + +### CLI Interface +Provides a user-friendly command-line interface for common Pipelex operations (e.g., `run`, `validate`, `status`). It acts as the entry point for command-line users by parsing commands and delegating to the core system. + + +**Related Classes/Methods**: + +- `pipelex.cli` + + +### Python Client/SDK +Offers a comprehensive Python API for developers to integrate Pipelex into their applications. It allows programmatic control over pipeline creation, execution, and monitoring, serving as a high-level API for defining and managing pipelines. + + +**Related Classes/Methods**: + +- `pipelex.client` + + +### Pipeline Tracker +Manages the lifecycle, status, and historical data of pipeline runs, including their structural definitions. It provides mechanisms for reporting and visualization of pipeline execution. + + +**Related Classes/Methods**: + +- `pipelex.pipeline.track` + + +### Activity Manager +Oversees and manages the individual steps or "activities" within a pipeline, including their input, output, and execution status. It provides granular details of pipeline execution. + + +**Related Classes/Methods**: + +- `pipelex.pipeline.activity` + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/User_Interface_System_Initialization.md b/docs/pages/advanced-customization/User_Interface_System_Initialization.md new file mode 100644 index 000000000..0bd6285c1 --- /dev/null +++ b/docs/pages/advanced-customization/User_Interface_System_Initialization.md @@ -0,0 +1,88 @@ +```mermaid +graph LR + Document_Ingestion["Document Ingestion"] + Text_Splitter["Text Splitter"] + Vector_Store["Vector Store"] + Embeddings_Model["Embeddings Model"] + Language_Model_LLM_["Language Model (LLM)"] + Retrieval_Chain["Retrieval Chain"] + Unclassified["Unclassified"] + Document_Ingestion -- "loads documents into" --> Text_Splitter + Text_Splitter -- "splits text for" --> Embeddings_Model + Embeddings_Model -- "generates embeddings for" --> Vector_Store + Vector_Store -- "stores embeddings from" --> Embeddings_Model + Vector_Store -- "retrieves context for" --> Retrieval_Chain + Retrieval_Chain -- "uses" --> Language_Model_LLM_ + Language_Model_LLM_ -- "answers queries using" --> Retrieval_Chain +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +This graph represents the core functionality of a document processing and question-answering system. The main flow involves ingesting documents, processing them into a searchable format, and then using a language model to answer user queries based on the ingested content. Its purpose is to provide an intelligent interface for users to retrieve information from a collection of documents. + +### Document Ingestion +Handles the loading and initial processing of various document types. + + +**Related Classes/Methods**: + +- `langchain_community.document_loaders.pdf.PyPDFLoader` +- `langchain_community.document_loaders.csv_loader.CSVLoader` + + +### Text Splitter +Breaks down large documents into smaller, manageable chunks for efficient processing and embedding. + + +**Related Classes/Methods**: + +- `langchain.text_splitter.RecursiveCharacterTextSplitter` + + +### Vector Store +Stores and retrieves document embeddings, enabling semantic search. + + +**Related Classes/Methods**: + +- `langchain_community.vectorstores.chroma.Chroma` + + +### Embeddings Model +Generates numerical representations (embeddings) of text chunks. + + +**Related Classes/Methods**: + +- `langchain_community.embeddings.ollama.OllamaEmbeddings` + + +### Language Model (LLM) +Processes user queries and generates answers based on retrieved context. + + +**Related Classes/Methods**: + +- `langchain_community.llms.ollama.Ollama` + + +### Retrieval Chain +Orchestrates the retrieval of relevant document chunks and passes them to the LLM for answer generation. + + +**Related Classes/Methods**: + +- `langchain.chains.retrieval.create_retrieval_chain` + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Workflow_Definition_Parsing.md b/docs/pages/advanced-customization/Workflow_Definition_Parsing.md new file mode 100644 index 000000000..f513f4b0c --- /dev/null +++ b/docs/pages/advanced-customization/Workflow_Definition_Parsing.md @@ -0,0 +1,54 @@ +```mermaid +graph LR + Workflow_Interpreter["Workflow Interpreter"] + Concept_Library_Manager["Concept Library Manager"] + Library_Manager["Library Manager"] + Unclassified["Unclassified"] + Workflow_Interpreter -- "depends on" --> Concept_Library_Manager + Concept_Library_Manager -- "provides schema definitions to" --> Workflow_Interpreter + Workflow_Interpreter -- "depends on" --> Library_Manager + Library_Manager -- "provides executable components to" --> Workflow_Interpreter +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The core of the system revolves around the `Workflow Interpreter`, which is responsible for processing PLX workflow definitions. It relies on the `Concept Library Manager` to access and validate data schemas ("concepts") used within these workflows, ensuring data consistency. Additionally, the `Workflow Interpreter` interacts with the `Library Manager` to discover and load "pipe" and "domain" libraries, which provide the concrete operational components for executing the defined workflows. This architecture emphasizes a clear separation of concerns, with dedicated components for parsing, data schema management, and library management, facilitating extensibility and maintainability. + +### Workflow Interpreter +This component is the core of the subsystem, responsible for parsing the declarative PLX workflow definition, validating its syntax, and translating it into an internal, executable representation. It acts as the primary implementation of the 'PLX Parser/Interpreter' pattern, understanding the flow, conditions, and steps defined in the DSL. + + +**Related Classes/Methods**: + +- `pipelex.core.interpreter.PipelexInterpreter`:43-1043 + + +### Concept Library Manager +Manages and provides access to "concepts," which are essentially data schemas defining the structure and types of data used within workflows. It ensures consistency and validity of data flowing through the pipes, aligning with the 'Knowledge/Data Management' aspect for data schemas. + + +**Related Classes/Methods**: + +- `pipelex.core.concepts.concept_library.ConceptLibrary`:19-125 + + +### Library Manager +Discovers, loads, and manages "pipe" and "domain" libraries. These libraries contain the concrete AI/ML operations and domain-specific functionalities that are referenced and utilized within the workflows. This component is vital for the project's 'Extensibility/Plugin-based Architecture'. + + +**Related Classes/Methods**: + +- `pipelex.libraries.library_manager.LibraryManager`:53-268 + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Workflow_Definition_Registry.md b/docs/pages/advanced-customization/Workflow_Definition_Registry.md new file mode 100644 index 000000000..33dc89fe5 --- /dev/null +++ b/docs/pages/advanced-customization/Workflow_Definition_Registry.md @@ -0,0 +1,66 @@ +```mermaid +graph LR + PLX_Interpreter["PLX Interpreter"] + Pipe_Registry["Pipe Registry"] + Concept_Registry["Concept Registry"] + Domain_Registry["Domain Registry"] + Unclassified["Unclassified"] + PLX_Interpreter -- "Registers definitions with" --> Pipe_Registry + PLX_Interpreter -- "Registers definitions with" --> Concept_Registry + PLX_Interpreter -- "Registers definitions with" --> Domain_Registry + Pipe_Registry -- "References" --> Concept_Registry + Pipe_Registry -- "Is categorized by" --> Domain_Registry + Concept_Registry -- "Is categorized by" --> Domain_Registry +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The `Workflow Definition & Registry` subsystem is responsible for the interpretation of the declarative PLX language and the management of all defined pipes, concepts (data schemas), and domains. It acts as the foundational layer for transforming high-level workflow descriptions into an executable internal representation and maintaining a consistent catalog of available components. + +### PLX Interpreter +This component is the primary entry point for processing raw PLX content. It performs lexical analysis, parsing, and semantic validation of pipeline, pipe, concept, and domain definitions. Its core responsibility is to transform these declarative definitions into an executable internal blueprint, registering them with the respective registries. + + +**Related Classes/Methods**: + +- `pipelex.core.interpreter.PipelexInterpreter`:30-1016 + + +### Pipe Registry +The Pipe Registry serves as the central repository for all user-defined and native pipe definitions. It manages their lifecycle, including registration, retrieval, and validation. This registry stores critical metadata such as input/output schemas, configuration parameters, and references to their execution logic, ensuring pipes are discoverable and consistently applied within pipelines. + + +**Related Classes/Methods**: + +- `pipelex.core.pipes` + + +### Concept Registry +This component is responsible for managing the lifecycle and availability of all user-defined and native concepts, which represent structured data schemas. It ensures data consistency and type safety across different pipes and pipelines by providing a centralized definition of data types and structures. + + +**Related Classes/Methods**: + +- `pipelex.core.concepts` + + +### Domain Registry +The Domain Registry manages the lifecycle and availability of all user-defined and native domains. Domains likely provide contextual categorization, scoping, or environmental definitions for pipes and concepts, facilitating organization and potentially access control within the PLX ecosystem. + + +**Related Classes/Methods**: + +- `pipelex.core.domains` + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Workflow_Execution_Engine.md b/docs/pages/advanced-customization/Workflow_Execution_Engine.md new file mode 100644 index 000000000..e3ba6c06d --- /dev/null +++ b/docs/pages/advanced-customization/Workflow_Execution_Engine.md @@ -0,0 +1,93 @@ +```mermaid +graph LR + Pipeline_Manager["Pipeline Manager"] + Pipe_Controller["Pipe Controller"] + Specialized_Pipe_Controllers["Specialized Pipe Controllers"] + Client_Interface["Client Interface"] + Working_Memory["Working Memory"] + Pipe_Operator["Pipe Operator"] + Hub["Hub"] + Client_Interface -- "requests" --> Hub + Hub -- "initiates" --> Pipeline_Manager + Pipeline_Manager -- "delegates to" --> Pipe_Controller + Pipe_Controller -- "dispatches to" --> Specialized_Pipe_Controllers + Pipe_Controller -- "manages state with" --> Working_Memory + Specialized_Pipe_Controllers -- "invokes" --> Pipe_Operator + click Working_Memory href "https://github.com/Pipelex/pipelex/blob/main/.codeboarding/Working_Memory.md" "Details" +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/GeneratedOnBoardings)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/demo)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The `pipelex` system is designed around a robust pipeline orchestration architecture. External clients interact via the `Client Interface`, which leverages the `Hub` to provision and manage `Pipeline Manager` instances. The `Pipeline Manager` orchestrates the high-level execution flow, delegating the detailed control of pipeline segments to the `Pipe Controller`. The `Pipe Controller` then dispatches to various `Specialized Pipe Controllers` (Sequence, Condition, Parallel, Batch) to implement specific control flow patterns. Throughout this process, `Working Memory` provides the necessary state and data context. The actual processing tasks are performed by `Pipe Operator` components, which are invoked by the `Specialized Pipe Controllers`, ensuring a clear separation of concerns between control flow and operational logic. + +### Pipeline Manager +Acts as the primary orchestrator, managing the overall lifecycle and execution flow of a pipeline from initiation to completion. It translates the high-level pipeline definition into executable steps. This component is crucial as the entry point for pipeline execution, embodying the "Orchestration" pattern. + + +**Related Classes/Methods**: + +- `pipelex.pipeline.pipeline_manager` + + +### Pipe Controller +Serves as a central dispatcher that routes execution to specific pipe control strategies. It manages the state and data flow for a segment of the pipeline, ensuring proper context and data exchange. This component is central to implementing the "Workflow Orchestrator" aspect, directing the flow based on pipeline structure. + + +**Related Classes/Methods**: + +- `pipelex.pipe_controllers.pipe_controller` + + +### Specialized Pipe Controllers +A group of specialized controllers (Sequence, Condition, Parallel, Batch) that implement specific control flow logic for executing pipes. They are dispatched by the `Pipe Controller` based on the pipeline's structure, handling sequential, conditional, parallel, or batch processing of pipes. + + +**Related Classes/Methods**: + +- `pipelex.pipe_controllers.sequence.pipe_sequence` +- `pipelex.pipe_controllers.condition.pipe_condition` +- `pipelex.pipe_controllers.parallel.pipe_parallel` +- `pipelex.pipe_controllers.batch.pipe_batch` + + +### Client Interface +Provides the external interface for interacting with the `pipelex` system, allowing users or other services to request and manage pipeline executions. It acts as the entry point for initiating pipeline operations. + + +**Related Classes/Methods**: + +- `pipelex.client.client.Client` + + +### Working Memory [[Expand]](./Working_Memory.md) +A central repository for storing and retrieving the current state, data, and context relevant to an ongoing pipeline execution. It ensures that all components have access to the necessary information at each step. + + +**Related Classes/Methods**: + +- `pipelex.core.memory.working_memory.WorkingMemory`:37-344 + + +### Pipe Operator +Encapsulates the atomic processing logic for a single step within a pipeline. Each operator performs a specific task, such as data transformation, external API calls, or AI model inference. + + +**Related Classes/Methods**: + +- `pipelex.pipe_operators.pipe_operator.PipeOperator`:16-90 + + +### Hub +Acts as a central registry and factory for pipeline-related services. It is responsible for providing access to and initiating core pipeline components, such as the `Pipeline Manager`, based on client requests. + + +**Related Classes/Methods**: + +- `pipelex.hub.Hub` + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/Working_Memory_Data_Store.md b/docs/pages/advanced-customization/Working_Memory_Data_Store.md new file mode 100644 index 000000000..7d2a529c8 --- /dev/null +++ b/docs/pages/advanced-customization/Working_Memory_Data_Store.md @@ -0,0 +1,54 @@ +```mermaid +graph LR + WorkingMemory["WorkingMemory"] + StuffFactory["StuffFactory"] + Stuff["Stuff"] + Unclassified["Unclassified"] + WorkingMemory -- "manages" --> Stuff + WorkingMemory -- "receives" --> Stuff + StuffFactory -- "produces" --> Stuff + StuffFactory -- "provides to" --> WorkingMemory +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The `pipelex` core subsystem is designed around the central concept of `Stuff` objects, which are the fundamental data units. The `StuffFactory` is responsible for the initial creation of these `Stuff` objects from diverse input sources, encapsulating them into a standardized format. Once created, these `Stuff` objects are provided to the `WorkingMemory`. The `WorkingMemory` then assumes the critical role of managing the in-memory state of these `Stuff` objects, handling their storage, retrieval, and overall lifecycle within the pipeline, thereby ensuring data consistency and accessibility for subsequent processing stages. This architecture facilitates a clear separation of concerns, with `StuffFactory` handling data ingestion and `WorkingMemory` managing the in-memory data state. + +### WorkingMemory +Manages the in-memory state of data ("stuffs") that flows through the pipeline. It acts as the central data bus, providing mechanisms for storing, retrieving, and managing the lifecycle of `Stuff` objects. It ensures data consistency and accessibility across different pipes. + + +**Related Classes/Methods**: + +- `pipelex.core.memory.working_memory.WorkingMemory`:37-344 + + +### StuffFactory +A factory for creating `Stuff` objects from various input formats (e.g., raw text, images, structured data). It ensures that data is correctly encapsulated into the `Stuff` representation for use within the `WorkingMemory`. + + +**Related Classes/Methods**: + +- `pipelex.core.stuffs.stuff_factory.StuffFactory`:38-261 + + +### Stuff +Represents the fundamental data unit within the pipeline. It is designed to encapsulate various content types (text, image, PDF, structured objects like JSON, HTML, Markdown) and provides methods for accessing and transforming its content (e.g., `get_stuff_as_text`, `get_stuff_as_image`). + + +**Related Classes/Methods**: + +- `pipelex.core.stuffs.stuff.Stuff`:26-194 + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/on_boarding.md b/docs/pages/advanced-customization/on_boarding.md new file mode 100644 index 000000000..7794db4f0 --- /dev/null +++ b/docs/pages/advanced-customization/on_boarding.md @@ -0,0 +1,116 @@ +```mermaid +graph LR + Pipelex_Core_User_Interface["Pipelex Core & User Interface"] + Pipeline_Definition_Data_Schema["Pipeline Definition & Data Schema"] + Workflow_Execution_Engine["Workflow Execution Engine"] + Pipe_Operators_Working_Memory["Pipe Operators & Working Memory"] + AI_Service_Integration_Layer["AI Service Integration Layer"] + Pipeline_Observability_Reporting["Pipeline Observability & Reporting"] + Pipelex_Core_User_Interface -- "initiates pipeline execution" --> Pipeline_Definition_Data_Schema + Pipeline_Observability_Reporting -- "provides execution reports" --> Pipelex_Core_User_Interface + Pipeline_Definition_Data_Schema -- "provides executable pipeline structure" --> Workflow_Execution_Engine + Pipeline_Definition_Data_Schema -- "defines data schemas" --> Pipe_Operators_Working_Memory + Workflow_Execution_Engine -- "orchestrates pipe execution" --> Pipe_Operators_Working_Memory + Workflow_Execution_Engine -- "sends execution events" --> Pipeline_Observability_Reporting + Pipe_Operators_Working_Memory -- "requests AI services" --> AI_Service_Integration_Layer + AI_Service_Integration_Layer -- "provides AI service responses" --> Pipe_Operators_Working_Memory + Pipe_Operators_Working_Memory -- "sends activity events" --> Pipeline_Observability_Reporting + click Pipelex_Core_User_Interface href "https://github.com/Pipelex/pipelex/blob/main/.codeboarding/Pipelex_Core_User_Interface.md" "Details" + click Pipeline_Definition_Data_Schema href "https://github.com/Pipelex/pipelex/blob/main/.codeboarding/Pipeline_Definition_Data_Schema.md" "Details" + click Workflow_Execution_Engine href "https://github.com/Pipelex/pipelex/blob/main/.codeboarding/Workflow_Execution_Engine.md" "Details" + click Pipe_Operators_Working_Memory href "https://github.com/Pipelex/pipelex/blob/main/.codeboarding/Pipe_Operators_Working_Memory.md" "Details" + click AI_Service_Integration_Layer href "https://github.com/Pipelex/pipelex/blob/main/.codeboarding/AI_Service_Integration_Layer.md" "Details" + click Pipeline_Observability_Reporting href "https://github.com/Pipelex/pipelex/blob/main/.codeboarding/Pipeline_Observability_Reporting.md" "Details" +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/GeneratedOnBoardings)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/demo)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The Pipelex architecture is designed around a workflow-centric, modular pipeline pattern, enabling declarative AI workflow automation. At its core, the Pipelex Core & User Interface serves as the central orchestrator, allowing users to define and initiate complex AI pipelines. These pipelines are structured by the Pipeline Definition & Data Schema, which translates declarative blueprints into executable workflows. The Workflow Execution Engine then takes charge, orchestrating the execution of individual Pipe Operators & Working Memory components. These operators perform the actual data processing, managing transient data within their working memory, and seamlessly integrating with various external AI services via the AI Service Integration Layer. Throughout the entire process, the Pipeline Observability & Reporting component provides comprehensive tracking and reporting, ensuring transparency and debuggability of the AI workflows. This design facilitates clear data and control flow, making it ideal for visual diagram generation that highlights the progression of data through defined stages and interactions with external AI capabilities. + +### Pipelex Core & User Interface [[Expand]](./Pipelex_Core_User_Interface.md) +The central application entry point and control unit, handling user interactions (CLI/Client) and orchestrating the overall system. It manages application initialization, configuration loading, and command handling. + + +**Related Classes/Methods**: + +- `pipelex.pipelex`:61-305 +- `pipelex.hub` +- `pipelex.cli.commands.__init__` +- `pipelex.client.client` + + +### Pipeline Definition & Data Schema [[Expand]](./Pipeline_Definition_Data_Schema.md) +Manages the declarative definition of pipelines (e.g., TOML parsing) and the underlying data structures (concepts, domains) that define the data flowing through the pipes. It translates blueprints into executable internal representations. + + +**Related Classes/Methods**: + +- `pipelex.core.interpreter` +- `pipelex.core.pipes.pipe_factory` +- `pipelex.core.pipes.pipe_library` +- `pipelex.libraries.library_manager` +- `pipelex.core.concepts.concept_library` +- `pipelex.core.domains.domain_library` + + +### Workflow Execution Engine [[Expand]](./Workflow_Execution_Engine.md) +The core engine that orchestrates the execution of individual pipes within a defined pipeline. It manages control flow logic such as sequencing, conditional branching, and parallel execution, ensuring pipes are executed in the correct order. + + +**Related Classes/Methods**: + +- `pipelex.pipe_controllers.pipe_controller` +- `pipelex.pipe_controllers.sequence.pipe_sequence` +- `pipelex.pipe_controllers.condition.pipe_condition` +- `pipelex.pipeline.pipeline_manager` + + +### Pipe Operators & Working Memory [[Expand]](./Pipe_Operators_Working_Memory.md) +Atomic, pluggable units of work that perform specific data processing tasks. These operators interact directly with an internal transient data store (working memory) and utilize templating for dynamic content generation and manipulation. + + +**Related Classes/Methods**: + +- `pipelex.pipe_operators.pipe_operator` +- `pipelex.pipe_operators.llm.pipe_llm` +- `pipelex.pipe_operators.ocr.pipe_ocr` +- `pipelex.pipe_operators.img_gen.pipe_img_gen` +- `pipelex.core.memory.working_memory` +- `pipelex.core.stuffs.stuff` +- `pipelex.core.stuffs.stuff_factory` +- `pipelex.tools.templating.jinja2_rendering` +- `pipelex.tools.templating.template_library` + + +### AI Service Integration Layer [[Expand]](./AI_Service_Integration_Layer.md) +Provides a unified and abstracted interface for interacting with various external AI/ML services (e.g., LLMs, Image Generation, OCR). It manages different AI provider adapters (plugins) and handles service requests, abstracting away third-party API specifics. + + +**Related Classes/Methods**: + +- `pipelex.cogt.inference.inference_manager` +- `pipelex.cogt.llm.llm_worker_abstract` +- `pipelex.cogt.imgg.imgg_worker_abstract` +- `pipelex.cogt.ocr.ocr_worker_abstract` +- `pipelex.plugins.plugin_manager` +- `pipelex.plugins.openai.openai_config` +- `pipelex.plugins.anthropic.anthropic_config` + + +### Pipeline Observability & Reporting [[Expand]](./Pipeline_Observability_Reporting.md) +Monitors pipeline execution, tracks activities, and generates reports and visualizations (e.g., flowcharts) for analysis, debugging, and operational insights. It provides the necessary data for understanding pipeline performance and behavior. + + +**Related Classes/Methods**: + +- `pipelex.pipeline.track.pipeline_tracker` +- `pipelex.pipeline.track.flow_chart` +- `pipelex.reporting.reporting_manager` +- `pipelex.pipeline.activity.activity_manager` + + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/docs/pages/advanced-customization/overview.md b/docs/pages/advanced-customization/overview.md new file mode 100644 index 000000000..a7159d371 --- /dev/null +++ b/docs/pages/advanced-customization/overview.md @@ -0,0 +1,107 @@ +```mermaid +graph LR + CLI_User_Interface["CLI & User Interface"] + Pipeline_Definition_Builder["Pipeline Definition & Builder"] + Core_Orchestration_Engine["Core Orchestration Engine"] + Pipe_Operators_Controllers["Pipe Operators & Controllers"] + AI_Inference_Model_Management["AI Inference & Model Management"] + Data_Memory_Management["Data & Memory Management"] + Reporting_Observability["Reporting & Observability"] + Unclassified["Unclassified"] + CLI_User_Interface -- "initiates" --> Pipeline_Definition_Builder + CLI_User_Interface -- "initiates" --> Core_Orchestration_Engine + Pipeline_Definition_Builder -- "outputs pipeline configuration to" --> Core_Orchestration_Engine + Core_Orchestration_Engine -- "orchestrates" --> Pipe_Operators_Controllers + Pipe_Operators_Controllers -- "interacts with" --> AI_Inference_Model_Management + Pipe_Operators_Controllers -- "reads from/writes to" --> Data_Memory_Management + Core_Orchestration_Engine -- "interacts with" --> Data_Memory_Management + Core_Orchestration_Engine -- "sends events to" --> Reporting_Observability + click CLI_User_Interface href "https://github.com/Pipelex/pipelex/blob/main/.codeboarding/CLI_User_Interface.md" "Details" + click Reporting_Observability href "https://github.com/Pipelex/pipelex/blob/main/.codeboarding/Reporting_Observability.md" "Details" +``` + +[![CodeBoarding](https://img.shields.io/badge/Generated%20by-CodeBoarding-9cf?style=flat-square)](https://github.com/CodeBoarding/CodeBoarding)[![Demo](https://img.shields.io/badge/Try%20our-Demo-blue?style=flat-square)](https://www.codeboarding.org/diagrams)[![Contact](https://img.shields.io/badge/Contact%20us%20-%20contact@codeboarding.org-lightgrey?style=flat-square)](mailto:contact@codeboarding.org) + +## Details + +The Pipelex project is structured around a core orchestration engine that manages the execution of AI pipelines. User interaction primarily occurs through a Command-Line Interface (CLI), which initiates pipeline building and execution. Pipeline definitions, written in a declarative PLX format, are processed by a dedicated builder component to create an executable flow. During execution, the orchestration engine dispatches tasks to various pipe operators and controllers, which interact with AI inference and model management services for AI-driven tasks. Data and memory management components handle the flow and storage of information throughout the pipeline. Finally, a reporting and observability component tracks pipeline execution and generates reports. + +### CLI & User Interface [[Expand]](./CLI_User_Interface.md) +Provides the command-line interface for users to interact with Pipelex, enabling them to run, build, validate, and manage pipelines. + + +**Related Classes/Methods**: + +- `pipelex.cli.PipelexCLI` +- `pipelex.cli.commands.run_cmd.run_cmd`:19-182 + + +### Pipeline Definition & Builder +Responsible for defining, parsing, and validating pipeline configurations (PLX files). It translates the declarative pipeline definitions into an executable flow graph. + + +**Related Classes/Methods**: + +- `pipelex.builder.flow_factory.FlowFactory`:22-119 +- `pipelex.language.plx_factory.PlxFactory`:28-336 + + +### Core Orchestration Engine +The central component that manages the entire lifecycle of a pipeline, from loading and execution to tracking and error handling. It orchestrates the flow of data and control between different pipe operators. + + +**Related Classes/Methods**: + +- `pipelex.Pipelex`:64-370 +- `pipelex.pipeline.pipeline_manager.PipelineManager`:12-43 + + +### Pipe Operators & Controllers +This component group encompasses the individual processing units (operators) and control flow mechanisms (controllers) within a pipeline. Operators perform specific AI-driven tasks, while controllers manage the execution logic. + + +**Related Classes/Methods**: + +- `pipelex.pipe_operators.pipe_operator.PipeOperator`:17-94 +- `pipelex.pipe_controllers.pipe_controller.PipeController`:14-86 + + +### AI Inference & Model Management +Provides a standardized interface for interacting with various external AI models and inference services. It abstracts away the complexities of different AI provider APIs and manages model configurations. + + +**Related Classes/Methods**: + +- `pipelex.cogt.inference.inference_manager.InferenceManager`:18-150 +- `pipelex.cogt.models.model_manager.ModelManager`:20-148 + + +### Data & Memory Management +Manages the flow, storage, and retrieval of data ("stuffs" and "concepts") within a running pipeline. This includes inputs, intermediate results, and final outputs. + + +**Related Classes/Methods**: + +- `pipelex.core.memory.working_memory.WorkingMemory`:42-385 +- `pipelex.core.stuffs.stuff_factory.StuffFactory`:38-487 + + +### Reporting & Observability [[Expand]](./Reporting_Observability.md) +Gathers and presents metrics and logs related to pipeline execution, offering insights into performance, resource usage, and potential issues. + + +**Related Classes/Methods**: + +- `pipelex.reporting.reporting_manager.ReportingManager`:29-123 +- `pipelex.observer.local_observer.LocalObserver`:10-35 + + +### Unclassified +Component for all unclassified files and utility functions (Utility functions/External Libraries/Dependencies) + + +**Related Classes/Methods**: _None_ + + + +### [FAQ](https://github.com/CodeBoarding/GeneratedOnBoardings/tree/main?tab=readme-ov-file#faq) diff --git a/mkdocs.yml b/mkdocs.yml index db8593c33..8bb8318eb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -148,6 +148,14 @@ nav: - Logging: pages/tools/logging.md - Advanced Customizations: - Overview: pages/advanced-customization/index.md + - Architecture Overview: pages/advanced-customization/on_boarding.md + - Client & API Layer: pages/advanced-customization/Client_API_Layer.md + - Core Orchestration Engine: pages/advanced-customization/Core_Orchestration_Engine.md + - AI Integration Layer: pages/advanced-customization/AI_Integration_Layer.md + - Pipe Operators: pages/advanced-customization/Pipe_Operators.md + - Data & Working Memory: pages/advanced-customization/Data_Working_Memory.md + - Configuration & Extensibility: pages/advanced-customization/Configuration_Extensibility_Management.md + - Observability & Reporting: pages/advanced-customization/Observability_Reporting.md - Secrets Provider: pages/advanced-customization/secrets-provider-injection.md - Reporting Delegate: pages/advanced-customization/reporting-delegate-injection.md - Pipeline Tracker: pages/advanced-customization/pipeline-tracker-injection.md