checkpoint: matches running, slight memory leak

CHANGELOG.md

@@ -1,5 +1,12 @@
 ## [1.2.5](https://github.com/ThinAirThings/uix/compare/v1.2.4...v1.2.5) (2024-05-23)
 
+## 2.4.0
+
+### Minor Changes
+
+- Added correct path resolution for config
+  Fixed type issue with setting MatchToRelationshipType
+
 ## 2.3.4
 
 ### Patch Changes

package.json

@@ -1,7 +1,7 @@
 {
     "name": "@thinairthings/uix",
     "author": "Dan Lannan",
-    "version": "2.3.4",
+    "version": "2.4.0",
     "type": "module",
     "types": "./dist/lib/index.d.ts",
     "bin": {
@@ -15,6 +15,7 @@
         "build": "tsup-node",
         "build:run": "tsup-node && node dist/cli/cli.js",
         "test": "dotenvx run -f .env.test -- pnpm build && dotenvx run -f .env.test -- vitest run basic --test-timeout=100000",
+        "test:match": "pnpm build && dotenvx run -f .env.test -- vitest run match --test-timeout=100000",
         "uix": "dotenvx run -f .env.test -- node ./dist/cli/cli.js --config=./tests/uix/uix.config.ts",
         "wipset": "changeset && changeset version && changeset publish"
     },

src/fns/deleteNodeFactory.ts

@@ -46,7 +46,7 @@ export const deleteNodeFactory = <
             ...nodeKey
         });
     }
-    console.log("Deleted", parentNodeKeys)
+    console.log("Deleted", nodeKey)
     if (!parentNodeKeys.length) return UixErr({
         subtype: UixErrSubtype.DELETE_NODE_FAILED,
         message: `Failed to delete node of type ${nodeKey.nodeType as string} with id ${nodeKey.nodeId}`,

src/templates/staticObjectsTemplate.ts

@@ -6,7 +6,7 @@ export const staticObjectsTemplate = (config: GenericUixConfig) => {
     return /* ts */`
 // Start of File
 import uixConfig from '${config.pathToConfig.replace('uix.config.ts', 'uix.config')}'
-import { NodeShape } from '@thinairthings/uix'
+import { NodeShape, NodeState } from '@thinairthings/uix'
 import neo4j from 'neo4j-driver'
 export type ConfiguredNodeTypeMap = typeof uixConfig.graph.nodeTypeMap
 
@@ -22,6 +22,10 @@ ${config.graph.nodeTypeMap['Root']
 ${Object.keys(config.graph.nodeTypeMap).map(nodeType =>
     /*ts*/`export type ${nodeType}Node = NodeShape<ConfiguredNodeTypeMap['${nodeType}']> \n`
         ).join('')}
+${Object.keys(config.graph.nodeTypeMap).map(nodeType =>
+    /*ts*/`export type ${nodeType}NodeState = NodeState<ConfiguredNodeTypeMap['${nodeType}']> \n`
+        ).join('')
+        }
 
 export const driver = neo4j.driver(
     process.env.NEO4J_URI!, 

src/vectors/upsertMatchToRelationship.ts → src/vectors/upsertMatchTypeEmbedding.ts

@@ -1,24 +1,47 @@
 import OpenAI from "openai"
-import { AnyNodeShape, GenericMatchToRelationshipType, GenericNodeShape, GenericNodeType } from "../types/NodeType"
+import { GenericNodeShape, GenericNodeType } from "../types/NodeType"
 import { Driver, EagerResult, Integer, Node } from "neo4j-driver"
 import { Ok, UixErr, UixErrSubtype } from "../types/Result"
 import { openAIAction } from "../clients/openai"
 import { neo4jAction } from "../clients/neo4j"
+import dedent from "dedent"
+import { GenericMatchToRelationshipType } from "../types/MatchToRelationshipType"
 
 
 
-export const upsertMatchToRelationship = async (
+export const upsertMatchTypeEmbedding = async (
     neo4jDriver: Driver,
     openaiClient: OpenAI,
-    nodeShape: AnyNodeShape,
-    matchToRelationshipType: GenericMatchToRelationshipType,
-    matchToNodeType: GenericNodeType
+    triggerNode: GenericNodeShape,
+    fromNodeType: GenericNodeType,
+    matchToRelationshipType: GenericMatchToRelationshipType
 ) => {
     // Try/catch this because you're not going to handle it with application logic.
     // You'll just log it.
     const result = await neo4jAction(openAIAction(async () => {
         // Create Node Type Summary
-        console.log("Creating Node Type Summary", nodeShape)
+        console.log("Creating Match Type Summary (nodeTypeEmbedding)", fromNodeType.type)
+        // Collect all nodes in weightedNodeTypeSet
+        const {
+            targetNode,
+            weightedNodeSet
+        } = await neo4jDriver.executeQuery<EagerResult<{
+            weightedNode: Node<Integer, GenericNodeShape>,
+            targetNode: Node<Integer, GenericNodeShape>
+        }>>(dedent/*cypher*/`
+            // Get the node that initiated the change as a reference point 
+            match (triggerNode:${triggerNode.nodeType} {nodeId: $triggerNode.nodeId})
+            // Get the parent node that is the target of this update
+            match (targetNode: ${fromNodeType.type})<-[:CHILD_TO|UNIQUE_TO*]-(triggerNode)
+            // Get all nodes in the weightedNodeTypeSet
+            match (targetNode)<-[:CHILD_TO|UNIQUE_TO*]-(weightedNode: ${matchToRelationshipType.weightedNodeTypeSet.map(({ NodeType }) => NodeType.type).join('|')})
+            return weightedNode, targetNode
+        `, {
+            triggerNode
+        }).then(res => ({
+            targetNode: res.records[0].get('targetNode').properties,
+            weightedNodeSet: res.records.map(record => record.get('weightedNode').properties)
+        }))
         const { type, description } = matchToRelationshipType
         const nodeTypeSummary = await openaiClient.chat.completions.create({
             model: 'gpt-4o',
@@ -34,43 +57,45 @@ export const upsertMatchToRelationship = async (
                         + `You should ignore information that is not relevant to the '${type.toUpperCase()}Type' paragraph as it was defined.\n`
                 }, {
                     role: 'user',
-                    content: `The JSON data to use is: ${JSON.stringify(nodeShape)}`
+                    content: `The JSON data to use is: ${JSON.stringify(Object.fromEntries(weightedNodeSet.map(node => ([node.nodeType, node]))))}`
                 }
             ]
         }).then(res => res.choices[0].message.content ?? '')
+        console.log("Target Node", targetNode)
+        console.log("Weighted Node Set", weightedNodeSet)
+        console.log("Node Type Summary", nodeTypeSummary)
         const nodeTypeEmbedding = await openaiClient.embeddings.create({
             model: 'text-embedding-3-large',
             input: nodeTypeSummary
         }).then(res => res.data[0].embedding)
         // Update Node
-        const nodeResult = await neo4jDriver.executeQuery<EagerResult<{
-            toNode: Node<Integer, GenericNodeShape>
+        const vectorNode = await neo4jDriver.executeQuery<EagerResult<{
+            vectorNode: Node<Integer, GenericNodeShape>
         }>>(/*cypher*/`
-            match (node:${nodeShape.nodeType} {nodeId: $nodeId})
-            merge (vectorNode:${nodeShape.nodeType}Vector:${matchToRelationshipType.type} {nodeId: $nodeId})-[:VECTOR_TO]->(node)
+            match (targetNode:${targetNode.nodeType} {nodeId: $targetNode.nodeId})
+            merge (vectorNode:${targetNode.nodeType}Vector:${matchToRelationshipType.type} {nodeId: $targetNode.nodeId})-[:VECTOR_TO]->(targetNode)
             on create
                 set vectorNode += $vectorNodeStructure
             on match 
                 set vectorNode += $vectorNodeStructure
-            with node
-            match(toNode: ${matchToNodeType.type})<-[:CHILD_TO|UNIQUE_TO*]-(node)
-            return toNode
+            return vectorNode
         `, {
-            nodeId: nodeShape.nodeId,
+            targetNode,
             vectorNodeStructure: {
                 nodeTypeSummary,
                 nodeTypeEmbedding
             }
-        }).then(res => res.records[0].get('toNode').properties)
-        if (!nodeResult) return UixErr({
+        }).then(res => res.records[0].get('vectorNode').properties)
+        console.log("Upserted Match Type Embedding", vectorNode)
+        if (!vectorNode) return UixErr({
             subtype: UixErrSubtype.UPDATE_NODE_FAILED,
             message: `Upsert match relationship error`,
             data: {
-                nodeShape,
+                targetNode,
                 matchToRelationshipType
             }
         });
-        return Ok(nodeResult)
+        return Ok(targetNode)
     }))()
     const { data, error } = result
     if (data) return data

src/vectors/upsertMatches.ts

@@ -1,6 +1,7 @@
 import { Driver } from "neo4j-driver";
-import { GenericMatchToRelationshipType, GenericNodeShape } from "../types/NodeType";
+import { GenericNodeShape } from "../types/NodeType";
 import dedent from 'dedent'
+import { GenericMatchToRelationshipType } from "../types/MatchToRelationshipType";
 
 
 

src/vectors/upsertMatchesv2.ts

@@ -0,0 +1,46 @@
+import { Driver, EagerResult, Integer, Node } from "neo4j-driver";
+import { GenericNodeShape } from "../types/NodeType";
+import { GenericMatchToRelationshipType } from "../types/MatchToRelationshipType";
+import dedent from "dedent";
+
+
+
+
+export const upsertMatchesv2 = async (
+    neo4jDriver: Driver,
+    fromNode: GenericNodeShape,
+    matchToRelationshipType: GenericMatchToRelationshipType
+) => {
+    // Run Query
+    const matches = await neo4jDriver.executeQuery<EagerResult<{
+        score: number,
+        matchToNode: Node<Integer, GenericNodeShape>
+    }>>(dedent/*cypher*/`
+        match (fromNode: ${fromNode.nodeType} {nodeId: $fromNode.nodeId})<-[:VECTOR_TO]-(fromNodeVectorNode:${fromNode.nodeType}Vector:${matchToRelationshipType.type})
+        call db.index.vector.queryNodes('${matchToRelationshipType.matchToNodeType.type}_vector', 5, fromNodeVectorNode.nodeTypeEmbedding)
+        yield node as matchToVectorNode, score
+        match (matchToNode: ${matchToRelationshipType.matchToNodeType.type} {nodeId: matchToVectorNode.nodeId})
+        merge (fromNode)-[match_to:MATCH_TO { type: $matchToType }]->(matchToNode)
+            on create
+                set
+                    match_to.fromNodeId = $fromNode.nodeId,
+                    match_to.type = $matchToType,
+                    match_to.toNodeId = matchToNode.nodeId,
+                    match_to.createdAt = timestamp(),
+                    match_to.updatedAt = timestamp(),
+                    match_to.score = score
+            on match
+                set match_to.updatedAt = timestamp(),
+                    match_to.score = score
+        return score, matchToNode  
+    `, {
+        fromNode,
+        matchToType: matchToRelationshipType.matchToNodeType.type
+    }).then(res => res.records.map(record => ({
+        score: record.get('score'),
+        matchToNode: record.get('matchToNode').properties
+    })))
+
+    console.log(matches)
+    return matches
+}

src/vectors/upsertVectorNode.ts

@@ -1,9 +1,11 @@
 import { Driver } from "neo4j-driver";
 import OpenAI from "openai";
 import { AnyNodeShape, GenericNodeType, GenericNodeTypeMap } from "../types/NodeType";
-import { upsertMatchToRelationship } from "./upsertMatchToRelationship";
 import { upsertPropertyVector } from "./upsertPropertyVector";
 import { upsertMatches } from "./upsertMatches";
+import { match } from "assert";
+import { upsertMatchTypeEmbedding } from "./upsertMatchTypeEmbedding";
+import { upsertMatchesv2 } from "./upsertMatchesv2";
 
 export const upsertVectorNode = async (
     neo4jDriver: Driver,
@@ -21,36 +23,48 @@ export const upsertVectorNode = async (
             nodeShape
         ))
     ])
-    // This gathers the matchToRelationshipTypes from the graph structure that are relevant to this specific node
-    const matchToRelationshipComponents = Object.values(nodeTypeMap).map(nodeType =>
-    ({
-        nodeType,
-        matchNodeTypes: nodeType.matchToRelationshipTypeSet.filter(matchToRelationshipType =>
-            [matchToRelationshipType.matchToNodeType.type, ...matchToRelationshipType.weightedNodeTypeSet.map(({ NodeType }) => NodeType.type)]
-                .some(matchNodeType => nodeShape.nodeType === matchNodeType)
-        ).flat()
-    })
-    )
-    // Handle Vector Operations
-    const toNodes = await Promise.all([
-        // Create Summary in background
-        ...matchToRelationshipComponents.length
-            ? matchToRelationshipComponents
-                .map(({ nodeType, matchNodeTypes }) => matchNodeTypes.flatMap(async matchNodeType => await upsertMatchToRelationship(
-                    neo4jDriver,
-                    openaiClient,
-                    nodeShape,
-                    matchNodeType,
-                    nodeType
-                ))).flat()
-            : [],
-    ])
-    // // Note: You need to figure out how to get a hold of the fromNode
-    if (!toNodes.length) return
-    await Promise.all(toNodes.map(async toNode => await upsertMatches(
-        neo4jDriver,
-        toNode!,
-        nodeTypeMap[toNode!.nodeType].matchToRelationshipTypeSet[0]
-    )))
+    // Check all possible fromNodeTypes to see which ones are associated with the nodeShape in question.
+    const fromNodeTypesAndRelationshipTypesToDirectlyUpdate = Object.values(nodeTypeMap).map(fromNodeType => ({
+        fromNodeType,
+        matchToRelationshipTypeSet: fromNodeType.matchToRelationshipTypeSet.filter(matchToRelationshipType =>
+            matchToRelationshipType.weightedNodeTypeSet.some(({ NodeType }) => NodeType.type === nodeShape.nodeType)
+        )
+    }))
+    console.log(fromNodeTypesAndRelationshipTypesToDirectlyUpdate)
+    const fromNodeTypesToUpdateMatches = Object.values(nodeTypeMap).map(fromNodeType => ({
+        fromNodeType,
+        matchToRelationshipTypeSet: fromNodeType.matchToRelationshipTypeSet.filter(matchToRelationshipType =>
+            matchToRelationshipType.weightedNodeTypeSet.some(({ NodeType }) => NodeType.type === nodeShape.nodeType)
+            || matchToRelationshipType.matchToNodeType.type === nodeShape.nodeType
+        )
+    }))
+    // Update all nodes that have a direct relationship to the nodeShape
+    const updatedTargetNodes = await Promise.all([
+        ...fromNodeTypesAndRelationshipTypesToDirectlyUpdate.map(async ({ fromNodeType, matchToRelationshipTypeSet }) =>
+            await Promise.all(matchToRelationshipTypeSet.map(async matchToRelationshipType => await upsertMatchTypeEmbedding(
+                neo4jDriver,
+                openaiClient,
+                nodeShape,
+                fromNodeType,
+                matchToRelationshipType
+            )))
+        )
+    ]).then(res => res.flat().filter(node => node !== undefined))
+    console.log("Updated Target Nodes", updatedTargetNodes)
+    // Add matches to targetNode
+    await Promise.all(updatedTargetNodes.map(async (targetNode) =>
+        await Promise.all(nodeTypeMap[targetNode.nodeType].matchToRelationshipTypeSet.map(async matchToRelationshipType => await upsertMatchesv2(
+            neo4jDriver,
+            targetNode,
+            matchToRelationshipType
+        )))
+    ))
 
+    // // // Note: You need to figure out how to get a hold of the fromNode
+    // if (!toNodes.length) return
+    // await Promise.all(toNodes.map(async toNode => await upsertMatches(
+    //     neo4jDriver,
+    //     toNode!,
+    //     nodeTypeMap[toNode!.nodeType].matchToRelationshipTypeSet[0]
+    // )))
 }

tests/basic.test.ts

@@ -45,18 +45,7 @@ test('Integration test', async () => {
         degree: 'Master of Arts (M.A.)',
         fieldOfStudy: 'Electrical Engineering',
     })
-    const educationNodeSet = Array.from({ length: 5 }).map(async (_, i) => await createNode(
-        [userNode],
-        'Education', {
-        school: 'RPI',
-        graduationYear: 2022,
-        description: 'I studied math',
-        degree: 'Master of Arts (M.A.)',
-        fieldOfStudy: 'Electrical Engineering',
-    }))
-    await Promise.all(educationNodeSet)
     expect(createdEducationNode).toBeTruthy()
-
     if (createEducationNodeError) {
         console.error(createEducationNodeError)
         expect(createEducationNodeError).toBeFalsy()
@@ -94,7 +83,6 @@ test('Integration test', async () => {
     expect(updatedEducationVectorNode).toBeTruthy()
     expect(createdEducationVectorNode.description![0]).not.toEqual(updatedEducationVectorNode.description![0])
 
-
     // Check getAllNodeType
     const { data: allEducationNodes, error: getAllEducationNodesError } = await getAllOfNodeType('Education', {
         limit: 4
@@ -135,7 +123,8 @@ test('Integration test', async () => {
         return
     }
     expect(userNodeByIndex).toBeTruthy()
-    await new Promise(res => setTimeout(res, 1000 * 20))
+    writeFileSync(path.resolve('tests', 'userNode.json'), JSON.stringify(userNodeByIndex, null, 2))
+    await new Promise(res => setTimeout(res, 1000 * 40))
     // Check deleteNode
     const { data: deleted, error: deleteError } = await deleteNode(userNodeByIndex)
     if (deleteError) {