WIP: synergy OOP refactoring

packages/api/src/conversation-subgroup/index.ts

@@ -1,4 +1,7 @@
 import { SDNAClass, SubjectEntity, SubjectFlag, SubjectProperty } from "@coasys/ad4m";
+import { findTopics } from "@coasys/flux-utils/src/synergy";
+import Topic, { TopicWithRelevance } from "../topic";
+import { SemanticRelationship } from "../semantic-relationship"
 
 @SDNAClass({
   name: "ConversationSubgroup",
@@ -25,4 +28,17 @@ export default class ConversationSubgroup extends SubjectEntity {
     required: false,
   })
   summary: string;
+
+  async topicsWithRelevance(): Promise<TopicWithRelevance[]> {
+    return findTopics(this.perspective, this.baseExpression)
+  }
+
+  async addTopicWithRelevance(topicName: string, relevance: number) {
+    let topic = await Topic.byName(this.perspective, topicName);
+    const relationship = new SemanticRelationship(this.perspective);
+    relationship.expression = this.baseExpression;
+    relationship.tag = topic.baseExpression;
+    relationship.relevance = relevance;
+    await relationship.save();
+  }
 }

packages/api/src/conversation/index.ts

@@ -1,4 +1,7 @@
 import { SDNAClass, SubjectEntity, SubjectFlag, SubjectProperty } from "@coasys/ad4m";
+import ConversationSubgroup from "../conversation-subgroup";
+import { findTopics, getAllTopics } from "@coasys/flux-utils/src/synergy";
+import Topic, { TopicWithRelevance } from "../topic";
 
 @SDNAClass({
   name: "Conversation",
@@ -25,4 +28,98 @@ export default class Conversation extends SubjectEntity {
     required: false,
   })
   summary: string;
+
+  async subgroups(): Promise<ConversationSubgroup[]> {
+    return await ConversationSubgroup.query(this.perspective, { source: this.baseExpression }) as ConversationSubgroup[];
+  }
+
+  async topicsWithRelevance(): Promise<TopicWithRelevance[]> {
+    const subgroups = await this.subgroups();
+    const allTopics = await Promise.all(
+      subgroups.map(subgroup => subgroup.topicsWithRelevance())
+    );
+    return allTopics.flat();
+  }
+
+  async processNewExpressions(unprocessedItems) {
+    // gather up data for LLM processing
+    const previousSubgroups = await this.subgroups();
+    const lastSubgroup = previousSubgroups[previousSubgroups.length - 1] as any;
+    const lastSubgroupTopics = await lastSubgroup?.topicsWithRelevance()
+    const lastSubgroupWithTopics = lastSubgroup ? { ...lastSubgroup, topics: lastSubgroupTopics } : null;
+    const existingTopics = await Topic.query(this.perspective) as Topic[];
+
+    // run LLM processing
+    const { conversationData, currentSubgroup, newSubgroup } = await LLMProcessing(
+      unprocessedItems,
+      previousSubgroups,
+      lastSubgroupWithTopics,
+      existingTopics
+    );
+
+    // update conversation text
+    this.conversationName = conversationData.name;
+    this.summary = conversationData.summary;
+    await this.update();
+
+    // gather up topics returned from LLM
+    //const allReturnedTopics = [];
+    //if (currentSubgroup) allReturnedTopics.push(...currentSubgroup.topics);
+    //if (newSubgroup) allReturnedTopics.push(...newSubgroup.topics);
+    //await Topic.ensureTopics(this.perspective, allReturnedTopics);
+
+    // update currentSubgroup if new data returned from LLM
+    if (currentSubgroup) {
+      lastSubgroup.subgroupName = currentSubgroup.name;
+      lastSubgroup.summary = currentSubgroup.summary;
+      for (const topic of currentSubgroup.topics) {
+        // skip topics already linked to the subgroup
+        if (lastSubgroupTopics.find((t) => t.name === topic.name)) continue;
+        await lastSubgroup.addTopicWithRelevance(topic.name, topic.relevance);
+      }
+    }
+
+
+    // create new subgroup if returned from LLM
+    let newSubgroupEntity;
+    if (newSubgroup) {
+      newSubgroupEntity = new ConversationSubgroup(this.perspective, undefined, this.baseExpression);
+      newSubgroupEntity.subgroupName = newSubgroup.name;
+      newSubgroupEntity.summary = newSubgroup.summary;
+      await newSubgroupEntity.save();
+      newSubgroupEntity = await newSubgroupEntity.get();
+      for (const topic of newSubgroup.topics) {
+        await newSubgroupEntity.addTopicWithRelevance(topic.name, topic.relevance);
+      }
+    }
+
+
+    // link items to subgroups
+    const indexOfFirstItemInNewSubgroup =
+      newSubgroup && unprocessedItems.findIndex((item) => item.baseExpression === newSubgroup.firstItemId);
+    for (const [itemIndex, item] of unprocessedItems.entries()) {
+      const itemsSubgroup = newSubgroup && itemIndex >= indexOfFirstItemInNewSubgroup ? newSubgroupEntity : lastSubgroup;
+
+      newLinks.push({
+        source: itemsSubgroup.baseExpression,
+        predicate: "ad4m://has_child",
+        target: item.baseExpression,
+      });
+    }
+    // create vector embeddings for each unprocessed item
+    await Promise.all(unprocessedItems.map((item) => createEmbedding(perspective, item.text, item.baseExpression)));
+    // update vector embedding for conversation
+    await removeEmbedding(perspective, conversation.baseExpression);
+    await createEmbedding(perspective, conversationData.summary, conversation.baseExpression);
+    // update vector embedding for currentSubgroup if returned from LLM
+    if (currentSubgroup) {
+      await removeEmbedding(perspective, lastSubgroup.baseExpression);
+      await createEmbedding(perspective, currentSubgroup.summary, lastSubgroup.baseExpression);
+    }
+    // create vector embedding for new subgroup if returned from LLM
+    if (newSubgroup) await createEmbedding(perspective, newSubgroup.summary, newSubgroupEntity.baseExpression);
+    // batch commit all new links (currently only "ad4m://has_child" links)
+    await perspective.addLinks(newLinks);
+    processing = false;
+  }
 }

packages/api/src/topic/index.ts

@@ -1,5 +1,12 @@
 import { SDNAClass, SubjectEntity, SubjectFlag, SubjectProperty } from "@coasys/ad4m";
 
+export class TopicWithRelevance {
+  baseExpression: string;
+  name: string;
+  relevance: number;
+}
+
+
 @SDNAClass({
   name: "Topic",
 })
@@ -16,4 +23,35 @@ export default class Topic extends SubjectEntity {
     resolveLanguage: "literal",
   })
   topic: string;
+
+  static async ensureTopics(perspective, topics: string[]): Promise<Topic[]> {
+    // Get existing topics
+    const existingTopics = await Topic.query(perspective) as Topic[];
+    const newTopicsToCreate = topics.reduce((acc, topic) => {
+      const unique = !acc.some((t) => t.topic === topic) && !existingTopics.some((t) => t.topic === topic);
+      if (unique) acc.push(topic);
+      return acc;
+    }, []);
+    // create new topics and store them in newTopics array for later use
+    const newTopics: Topic[] = [];
+    for (const topic of newTopicsToCreate) {
+      // create topic
+      const newTopic = new Topic(perspective);
+      newTopic.topic = topic;
+      await newTopic.save();
+      newTopics.push(await newTopic.get());
+    }
+    return newTopics
+  }
+
+  static async byName(perspective, topicName: string): Promise<Topic> {
+    const existingTopics = await Topic.query(perspective) as Topic[];
+    const existingTopic = existingTopics.find(topic => topic.topic === topicName);
+    if (existingTopic) return existingTopic;
+
+    const newTopic = new Topic(perspective);
+    newTopic.topic = topicName;
+    await newTopic.save();
+    return await newTopic.get();
+  }
 }

packages/utils/src/synergy.ts

@@ -22,6 +22,7 @@ import JSON5 from "json5";
 import { v4 as uuidv4 } from "uuid";
 import { synergyGroupingPrompt, synergyGroupingExamples } from "./synergy-prompts";
 import { sleep } from "./sleep";
+import { TopicWithRelevance } from "@coasys/flux-api/src/topic";
 
 async function removeEmbedding(perspective, itemId) {
   const allSemanticRelationships = (await SemanticRelationship.query(perspective, {
@@ -140,12 +141,12 @@ export function transformItem(type, item) {
   return newItem;
 }
 
-export async function findTopics(perspective, itemId) {
+export async function findTopics(perspective, itemId): Promise<TopicWithRelevance[]> {
   const allRelationships = (await SemanticRelationship.query(perspective, {
     source: itemId,
   })) as any;
 
-  const topics = [];
+  const topics: TopicWithRelevance[] = [];
   for (const rel of allRelationships) {
     if (!rel.relevance) continue;
 
@@ -358,7 +359,7 @@ async function findOrCreateNewConversation(perspective: PerspectiveProxy, channe
 // + let other agents know when you have started & finished processing (add new signal in responsibleForProcessing check?)
 // + mark individual items as processing in UI
 let processing = false;
-async function processItemsAndAddToConversation(perspective, channelId, unprocessedItems) {
+async function processItemsAndAddToConversation(perspective, channelId, ) {
   processing = true;
   const conversation: any = await findOrCreateNewConversation(perspective, channelId);
   // gather up all new perspective links so they can be commited in a single transaction at the end of the function