add: reduce postprocessor

examples/chatgpt-filesearch.yaml

@@ -8,8 +8,35 @@ flows:
           chunkOverlap: 400
     ingestion:
       - filetypes: ["*"]
-        textsplitter:
-          name: text
+    retrieval:
+      querymodifiers:
+        # Enhance
+        - name: enhance
+          options:
+            model:
+              openai:
+                apiKey: "${OPENAI_API_KEY}"
+                model: gpt-4o
+                apiType: OPEN_AI
+                apiBase: https://api.openai.com/v1
+      retriever:
+        name: subquery
+        options:
+          limit: 3 # max. 3 subqueries
+          topK: 10 # topK per search
+          model:
+            openai:
+              apiKey: "${OPENAI_API_KEY}"
+              model: gpt-4o
+              apiType: OPEN_AI
+              apiBase: https://api.openai.com/v1
+      postprocessors:
+        - name: similarity
+          options:
+            threshold: 0.6
+        - name: reduce
           options:
-            chunkSize: 811
+            topK: 5
+
+
 

pkg/datastore/postprocessors/postprocessors.go

@@ -46,6 +46,7 @@ var PostprocessorMap = map[string]Postprocessor{
 	ContentSubstringFilterPostprocessorName:      &ContentSubstringFilterPostprocessor{},
 	ContentFilterPostprocessorName:               &ContentFilterPostprocessor{},
 	CohereRerankPostprocessorName:                &CohereRerankPostprocessor{},
+	ReducePostprocessorName:                      &ReducePostprocessor{},
 }
 
 func GetPostprocessor(name string) (Postprocessor, error) {

pkg/datastore/postprocessors/reduce.go

@@ -0,0 +1,41 @@
+package postprocessors
+
+import (
+	"context"
+	"github.com/gptscript-ai/knowledge/pkg/datastore/types"
+	vs "github.com/gptscript-ai/knowledge/pkg/vectorstore"
+	"slices"
+)
+
+const ReducePostprocessorName = "reduce"
+
+type ReducePostprocessor struct {
+	TopK int
+}
+
+func (s *ReducePostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error {
+	for q, docs := range response.Responses {
+
+		slices.SortFunc(docs, func(i, j vs.Document) int {
+			if i.SimilarityScore > j.SimilarityScore {
+				return -1
+			}
+			if i.SimilarityScore < j.SimilarityScore {
+				return 1
+			}
+			return 0
+		})
+
+		topK := s.TopK
+		if topK > len(docs) {
+			topK = len(docs) - 1
+		}
+
+		response.Responses[q] = docs[:topK]
+	}
+	return nil
+}
+
+func (s *ReducePostprocessor) Name() string {
+	return ReducePostprocessorName
+}