feat: add pinecone store support (#28)

Author: chr-hertel

README.md

@@ -40,22 +40,27 @@ llm_chain:
         original_embeddings:
             platform: 'openai'
     stores:
-        chroma_db:
-            engine: 'chroma-db'
-            collection_name: '%env(CHROMA_COLLECTION)%'
         azure_search:
             engine: 'azure-search'
-            api_key: '%env(AZURE_SEARCH_KEY)%' 
+            api_key: '%env(AZURE_SEARCH_KEY)%'
             endpoint: '%env(AZURE_SEARCH_ENDPOINT)%'
-            index_name: '%env(AZURE_SEARCH_INDEX)%' 
+            index_name: '%env(AZURE_SEARCH_INDEX)%'
             api_version: '2024-07-01'
+        chroma_db:
+            engine: 'chroma-db'
+            collection_name: '%env(CHROMA_COLLECTION)%'
         mongodb:
             engine: 'mongodb'
             database_name: '%env(MONGODB_DATABASE)%'
             collection_name: '%env(MONGODB_COLLECTION)%'
             index_name: '%env(MONGODB_INDEX)%'
             vector_field_name: 'vector'
             bulk_write: false
+        pinecone:
+            engine: 'pinecone'
+            namespace: 'partition'
+            filter: { 'key' : 'value' }
+            top_k: 5
 ```
 
 ## Usage

src/DependencyInjection/Configuration.php

@@ -52,15 +52,23 @@ public function getConfigTreeBuilder(): TreeBuilder
                     ->useAttributeAsKey('name')
                     ->arrayPrototype()
                         ->children()
-                            ->enumNode('engine')->values(['chroma-db', 'azure-search', 'mongodb'])->isRequired()->end()
-                            ->scalarNode('collection_name')->end()
-                            ->scalarNode('api_key')->end()
-                            ->scalarNode('endpoint')->end()
+                            ->enumNode('engine')->values(['azure-search', 'chroma-db', 'mongodb', 'pinecone'])->isRequired()->end()
+                            // Azure AI Search & MongoDB
                             ->scalarNode('index_name')->end()
+                            // Azure AI Search
+                            ->scalarNode('api_key')->end()
                             ->scalarNode('api_version')->end()
+                            ->scalarNode('endpoint')->end()
+                            // ChromaDB & MongoDB
+                            ->scalarNode('collection_name')->end()
+                            // MongoDB
                             ->scalarNode('database_name')->end()
-                            ->scalarNode('vector_field_name')->end()
                             ->booleanNode('bulk_write')->end()
+                            ->scalarNode('vector_field_name')->end()
+                            // Pinecone
+                            ->arrayNode('filter')->end()
+                            ->scalarNode('namespace')->end()
+                            ->scalarNode('top_k')->end()
                         ->end()
                     ->end()
             ->end()

src/DependencyInjection/LlmChainExtension.php

@@ -14,6 +14,7 @@
 use PhpLlm\LlmChain\Store\Azure\SearchStore as AzureSearchStore;
 use PhpLlm\LlmChain\Store\ChromaDB\Store as ChromaDBStore;
 use PhpLlm\LlmChain\Store\MongoDB\Store as MongoDBStore;
+use PhpLlm\LlmChain\Store\Pinecone\Store as PineconeStore;
 use PhpLlm\LlmChain\Store\StoreInterface;
 use PhpLlm\LlmChain\Store\VectorStoreInterface;
 use PhpLlm\LlmChain\ToolBox\AsTool;
@@ -148,13 +149,6 @@ private function processEmbeddingsConfig(string $name, array $embeddings, Contai
      */
     private function processStoreConfig(string $name, array $stores, ContainerBuilder $container): void
     {
-        if ('chroma-db' === $stores['engine']) {
-            $definition = new ChildDefinition(ChromaDBStore::class);
-            $definition->replaceArgument('$collectionName', $stores['collection_name']);
-
-            $container->setDefinition('llm_chain.store.'.$name, $definition);
-        }
-
         if ('azure-search' === $stores['engine']) {
             $definition = new ChildDefinition(AzureSearchStore::class);
             $definition
@@ -166,6 +160,13 @@ private function processStoreConfig(string $name, array $stores, ContainerBuilde
             $container->setDefinition('llm_chain.store.'.$name, $definition);
         }
 
+        if ('chroma-db' === $stores['engine']) {
+            $definition = new ChildDefinition(ChromaDBStore::class);
+            $definition->replaceArgument('$collectionName', $stores['collection_name']);
+
+            $container->setDefinition('llm_chain.store.'.$name, $definition);
+        }
+
         if ('mongodb' === $stores['engine']) {
             $definition = new ChildDefinition(MongoDBStore::class);
             $definition
@@ -177,5 +178,15 @@ private function processStoreConfig(string $name, array $stores, ContainerBuilde
 
             $container->setDefinition('llm_chain.store.'.$name, $definition);
         }
+
+        if ('pinecone' === $stores['engine']) {
+            $definition = new ChildDefinition(PineconeStore::class);
+            $definition
+                ->replaceArgument('$namespace', $stores['namespace'] ?? null)
+                ->replaceArgument('$filter', $stores['filter'] ?? [])
+                ->replaceArgument('$topK', $stores['top_k'] ?? 3);
+
+            $container->setDefinition('llm_chain.store.'.$name, $definition);
+        }
     }
 }

src/Resources/config/services.php

@@ -5,6 +5,8 @@
 namespace Symfony\Component\DependencyInjection\Loader\Configurator;
 
 use PhpLlm\LlmChain\Chain;
+use PhpLlm\LlmChain\Chain\InputProcessor;
+use PhpLlm\LlmChain\Chain\OutputProcessor;
 use PhpLlm\LlmChain\DocumentEmbedder;
 use PhpLlm\LlmChain\OpenAI\Model\Embeddings;
 use PhpLlm\LlmChain\OpenAI\Model\Gpt;
@@ -14,6 +16,11 @@
 use PhpLlm\LlmChain\Store\Azure\SearchStore as AzureSearchStore;
 use PhpLlm\LlmChain\Store\ChromaDB\Store as ChromaDBStore;
 use PhpLlm\LlmChain\Store\MongoDB\Store as MongoDBStore;
+use PhpLlm\LlmChain\Store\Pinecone\Store as PineconeStore;
+use PhpLlm\LlmChain\StructuredOutput\ChainProcessor as StructureOutputProcessor;
+use PhpLlm\LlmChain\StructuredOutput\ResponseFormatFactory;
+use PhpLlm\LlmChain\StructuredOutput\SchemaFactory;
+use PhpLlm\LlmChain\ToolBox\ChainProcessor as ToolProcessor;
 use PhpLlm\LlmChain\ToolBox\ParameterAnalyzer;
 use PhpLlm\LlmChain\ToolBox\ToolAnalyzer;
 use PhpLlm\LlmChain\ToolBox\ToolBox;
@@ -26,9 +33,17 @@
         ->defaults()
             ->autowire()
             ->autoconfigure()
+        ->instanceof(InputProcessor::class)
+            ->tag('llm_chain.chain.input_processor')
+        ->instanceof(OutputProcessor::class)
+            ->tag('llm_chain.chain.output_processor')
 
         // high level feature
         ->set(Chain::class)
+            ->args([
+                '$inputProcessor' => tagged_iterator('llm_chain.chain.input_processor'),
+                '$outputProcessor' => tagged_iterator('llm_chain.chain.output_processor'),
+            ])
         ->set(DocumentEmbedder::class)
 
         // platforms
@@ -81,6 +96,19 @@
                 '$vectorFieldName' => abstract_arg('The name of the field int the index that contains the vector'),
                 '$bulkWrite' => abstract_arg('Use bulk write operations'),
             ])
+        ->set(PineconeStore::class)
+            ->abstract()
+            ->args([
+                '$namespace' => abstract_arg('Namespace of index'),
+                '$filter' => abstract_arg('Filter for metadata'),
+                '$topK' => abstract_arg('Number of results to return'),
+            ])
+
+        // structured output
+        ->set(ResponseFormatFactory::class)
+        ->set(SchemaFactory::class)
+            ->factory([SchemaFactory::class, 'create'])
+        ->set(StructureOutputProcessor::class)
 
         // tools
         ->set(ToolBox::class)
@@ -90,6 +118,7 @@
             ->alias(ToolBoxInterface::class, ToolBox::class)
         ->set(ToolAnalyzer::class)
         ->set(ParameterAnalyzer::class)
+        ->set(ToolProcessor::class)
 
         // profiler
         ->set(DataCollector::class)