Merge pull request #1188 from FlowiseAI/feature/OpenAI-Assistant

Feature/Add openai assistant

packages/components/nodes/agents/OpenAIAssistant/OpenAIAssistant.ts

@@ -0,0 +1,224 @@
+import { ICommonObject, IDatabaseEntity, INode, INodeData, INodeOptionsValue, INodeParams } from '../../../src/Interface'
+import OpenAI from 'openai'
+import { DataSource } from 'typeorm'
+import { getCredentialData, getCredentialParam, getUserHome } from '../../../src/utils'
+import { MessageContentImageFile, MessageContentText } from 'openai/resources/beta/threads/messages/messages'
+import * as fsDefault from 'node:fs'
+import * as path from 'node:path'
+import fetch from 'node-fetch'
+
+class OpenAIAssistant_Agents implements INode {
+    label: string
+    name: string
+    version: number
+    description: string
+    type: string
+    icon: string
+    category: string
+    baseClasses: string[]
+    inputs: INodeParams[]
+
+    constructor() {
+        this.label = 'OpenAI Assistant'
+        this.name = 'openAIAssistant'
+        this.version = 1.0
+        this.type = 'OpenAIAssistant'
+        this.category = 'Agents'
+        this.icon = 'openai.png'
+        this.description = `An agent that uses OpenAI Assistant API to pick the tool and args to call`
+        this.baseClasses = [this.type]
+        this.inputs = [
+            {
+                label: 'Select Assistant',
+                name: 'selectedAssistant',
+                type: 'asyncOptions',
+                loadMethod: 'listAssistants'
+            }
+        ]
+    }
+
+    //@ts-ignore
+    loadMethods = {
+        async listAssistants(_: INodeData, options: ICommonObject): Promise<INodeOptionsValue[]> {
+            const returnData: INodeOptionsValue[] = []
+
+            const appDataSource = options.appDataSource as DataSource
+            const databaseEntities = options.databaseEntities as IDatabaseEntity
+
+            if (appDataSource === undefined || !appDataSource) {
+                return returnData
+            }
+
+            const assistants = await appDataSource.getRepository(databaseEntities['Assistant']).find()
+
+            for (let i = 0; i < assistants.length; i += 1) {
+                const assistantDetails = JSON.parse(assistants[i].details)
+                const data = {
+                    label: assistantDetails.name,
+                    name: assistants[i].id,
+                    description: assistantDetails.instructions
+                } as INodeOptionsValue
+                returnData.push(data)
+            }
+            return returnData
+        }
+    }
+
+    async init(): Promise<any> {
+        return null
+    }
+
+    async clearSessionMemory(nodeData: INodeData, options: ICommonObject): Promise<void> {
+        const selectedAssistantId = nodeData.inputs?.selectedAssistant as string
+        const appDataSource = options.appDataSource as DataSource
+        const databaseEntities = options.databaseEntities as IDatabaseEntity
+        let sessionId = nodeData.inputs?.sessionId as string
+
+        const assistant = await appDataSource.getRepository(databaseEntities['Assistant']).findOneBy({
+            id: selectedAssistantId
+        })
+
+        if (!assistant) throw new Error(`Assistant ${selectedAssistantId} not found`)
+
+        if (!sessionId && options.chatId) {
+            const chatmsg = await appDataSource.getRepository(databaseEntities['ChatMessage']).findOneBy({
+                chatId: options.chatId
+            })
+            if (!chatmsg) throw new Error(`Chat Message with Chat Id: ${options.chatId} not found`)
+            sessionId = chatmsg.sessionId
+        }
+
+        const credentialData = await getCredentialData(assistant.credential ?? '', options)
+        const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData)
+        if (!openAIApiKey) throw new Error(`OpenAI ApiKey not found`)
+
+        const openai = new OpenAI({ apiKey: openAIApiKey })
+        options.logger.info(`Clearing OpenAI Thread ${sessionId}`)
+        await openai.beta.threads.del(sessionId)
+        options.logger.info(`Successfully cleared OpenAI Thread ${sessionId}`)
+    }
+
+    async run(nodeData: INodeData, input: string, options: ICommonObject): Promise<string | object> {
+        const selectedAssistantId = nodeData.inputs?.selectedAssistant as string
+        const appDataSource = options.appDataSource as DataSource
+        const databaseEntities = options.databaseEntities as IDatabaseEntity
+
+        const assistant = await appDataSource.getRepository(databaseEntities['Assistant']).findOneBy({
+            id: selectedAssistantId
+        })
+
+        if (!assistant) throw new Error(`Assistant ${selectedAssistantId} not found`)
+
+        const credentialData = await getCredentialData(assistant.credential ?? '', options)
+        const openAIApiKey = getCredentialParam('openAIApiKey', credentialData, nodeData)
+        if (!openAIApiKey) throw new Error(`OpenAI ApiKey not found`)
+
+        const openai = new OpenAI({ apiKey: openAIApiKey })
+
+        // Retrieve assistant
+        const assistantDetails = JSON.parse(assistant.details)
+        const openAIAssistantId = assistantDetails.id
+        const retrievedAssistant = await openai.beta.assistants.retrieve(openAIAssistantId)
+
+        const chatmessage = await appDataSource.getRepository(databaseEntities['ChatMessage']).findOneBy({
+            chatId: options.chatId
+        })
+
+        let threadId = ''
+        if (!chatmessage) {
+            const thread = await openai.beta.threads.create({})
+            threadId = thread.id
+        } else {
+            const thread = await openai.beta.threads.retrieve(chatmessage.sessionId)
+            threadId = thread.id
+        }
+
+        // Add message to thread
+        await openai.beta.threads.messages.create(threadId, {
+            role: 'user',
+            content: input
+        })
+
+        // Run assistant thread
+        const runThread = await openai.beta.threads.runs.create(threadId, {
+            assistant_id: retrievedAssistant.id
+        })
+
+        const promise = (threadId: string, runId: string) => {
+            return new Promise((resolve, reject) => {
+                const timeout = setInterval(async () => {
+                    const run = await openai.beta.threads.runs.retrieve(threadId, runId)
+                    const state = run.status
+                    if (state === 'completed') {
+                        clearInterval(timeout)
+                        resolve(run)
+                    } else if (state === 'cancelled' || state === 'expired' || state === 'failed') {
+                        clearInterval(timeout)
+                        reject(new Error(`Error processing thread: ${state}, Thread ID: ${threadId}, Run ID: ${runId}`))
+                    }
+                }, 500)
+            })
+        }
+
+        // Polling run status
+        await promise(threadId, runThread.id)
+
+        // List messages
+        const messages = await openai.beta.threads.messages.list(threadId)
+        const messageData = messages.data ?? []
+        const assistantMessages = messageData.filter((msg) => msg.role === 'assistant')
+        if (!assistantMessages.length) return ''
+
+        let returnVal = ''
+        for (let i = 0; i < assistantMessages[0].content.length; i += 1) {
+            if (assistantMessages[0].content[i].type === 'text') {
+                const content = assistantMessages[0].content[i] as MessageContentText
+                returnVal += content.text.value
+
+                //TODO: handle annotations
+            } else {
+                const content = assistantMessages[0].content[i] as MessageContentImageFile
+                const fileId = content.image_file.file_id
+                const fileObj = await openai.files.retrieve(fileId)
+                const filePath = path.join(getUserHome(), '.flowise', 'openai-assistant', `${fileObj.filename}.png`)
+
+                await downloadFile(fileObj, filePath, openAIApiKey)
+
+                const bitmap = fsDefault.readFileSync(filePath)
+                const base64String = Buffer.from(bitmap).toString('base64')
+
+                const imgHTML = `<img src="data:image/png;base64,${base64String}" width="100%" height="max-content" alt="${fileObj.filename}" /><br/>`
+                returnVal += imgHTML
+            }
+        }
+
+        return { text: returnVal, assistant: { assistantId: openAIAssistantId, threadId, runId: runThread.id, messages: messageData } }
+    }
+}
+
+const downloadFile = async (fileObj: any, filePath: string, openAIApiKey: string) => {
+    try {
+        const response = await fetch(`https://api.openai.com/v1/files/${fileObj.id}/content`, {
+            method: 'GET',
+            headers: { Accept: '*/*', Authorization: `Bearer ${openAIApiKey}` }
+        })
+
+        if (!response.ok) {
+            throw new Error(`HTTP error! status: ${response.status}`)
+        }
+
+        await new Promise<void>((resolve, reject) => {
+            const dest = fsDefault.createWriteStream(filePath)
+            response.body.pipe(dest)
+            response.body.on('end', () => resolve())
+            dest.on('error', reject)
+        })
+
+        // eslint-disable-next-line no-console
+        console.log('File downloaded and written to', filePath)
+    } catch (error) {
+        console.error('Error downloading or writing the file:', error)
+    }
+}
+
+module.exports = { nodeClass: OpenAIAssistant_Agents }

packages/components/nodes/chatmodels/ChatOpenAI/ChatOpenAI.ts

@@ -47,6 +47,14 @@ class ChatOpenAI_ChatModels implements INode {
                         label: 'gpt-4',
                         name: 'gpt-4'
                     },
+                    {
+                        label: 'gpt-4-1106-preview',
+                        name: 'gpt-4-1106-preview'
+                    },
+                    {
+                        label: 'gpt-4-vision-preview',
+                        name: 'gpt-4-vision-preview'
+                    },
                     {
                         label: 'gpt-4-0613',
                         name: 'gpt-4-0613'
@@ -63,6 +71,10 @@ class ChatOpenAI_ChatModels implements INode {
                         label: 'gpt-3.5-turbo',
                         name: 'gpt-3.5-turbo'
                     },
+                    {
+                        label: 'gpt-3.5-turbo-1106',
+                        name: 'gpt-3.5-turbo-1106'
+                    },
                     {
                         label: 'gpt-3.5-turbo-0613',
                         name: 'gpt-3.5-turbo-0613'

packages/components/package.json

@@ -60,6 +60,7 @@
         "node-html-markdown": "^1.3.0",
         "notion-to-md": "^3.1.1",
         "object-hash": "^3.0.0",
+        "openai": "^4.16.1",
         "pdf-parse": "^1.1.1",
         "pdfjs-dist": "^3.7.107",
         "pg": "^8.11.2",

packages/server/src/Interface.ts

@@ -48,6 +48,15 @@ export interface ITool {
     createdDate: Date
 }
 
+export interface IAssistant {
+    id: string
+    details: string
+    credential: string
+    iconSrc?: string
+    updatedDate: Date
+    createdDate: Date
+}
+
 export interface ICredential {
     id: string
     name: string

packages/server/src/database/entities/Assistant.ts

@@ -0,0 +1,24 @@
+/* eslint-disable */
+import { Entity, Column, CreateDateColumn, UpdateDateColumn, PrimaryGeneratedColumn } from 'typeorm'
+import { IAssistant } from '../../Interface'
+
+@Entity()
+export class Assistant implements IAssistant {
+    @PrimaryGeneratedColumn('uuid')
+    id: string
+
+    @Column({ type: 'text' })
+    details: string
+
+    @Column()
+    credential: string
+
+    @Column({ nullable: true })
+    iconSrc?: string
+
+    @CreateDateColumn()
+    createdDate: Date
+
+    @UpdateDateColumn()
+    updatedDate: Date
+}

packages/server/src/database/entities/index.ts

@@ -2,10 +2,12 @@ import { ChatFlow } from './ChatFlow'
 import { ChatMessage } from './ChatMessage'
 import { Credential } from './Credential'
 import { Tool } from './Tool'
+import { Assistant } from './Assistant'
 
 export const entities = {
     ChatFlow,
     ChatMessage,
     Credential,
-    Tool
+    Tool,
+    Assistant
 }

packages/server/src/database/migrations/mysql/1699325775451-AddAssistantEntity.ts

@@ -0,0 +1,21 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddAssistantEntity1699325775451 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS \`assistant\` (
+                \`id\` varchar(36) NOT NULL,
+                \`credential\` varchar(255) NOT NULL,
+                \`details\` text NOT NULL,
+                \`iconSrc\` varchar(255) DEFAULT NULL,
+                \`createdDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+                \`updatedDate\` datetime(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
+                PRIMARY KEY (\`id\`)
+              ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;`
+        )
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE assistant`)
+    }
+}

packages/server/src/database/migrations/mysql/index.ts

@@ -6,6 +6,7 @@ import { ModifyTool1694001465232 } from './1694001465232-ModifyTool'
 import { AddApiConfig1694099200729 } from './1694099200729-AddApiConfig'
 import { AddAnalytic1694432361423 } from './1694432361423-AddAnalytic'
 import { AddChatHistory1694658767766 } from './1694658767766-AddChatHistory'
+import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEntity'
 
 export const mysqlMigrations = [
     Init1693840429259,
@@ -15,5 +16,6 @@ export const mysqlMigrations = [
     ModifyTool1694001465232,
     AddApiConfig1694099200729,
     AddAnalytic1694432361423,
-    AddChatHistory1694658767766
+    AddChatHistory1694658767766,
+    AddAssistantEntity1699325775451
 ]

packages/server/src/database/migrations/postgres/1699325775451-AddAssistantEntity.ts

@@ -0,0 +1,21 @@
+import { MigrationInterface, QueryRunner } from 'typeorm'
+
+export class AddAssistantEntity1699325775451 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(
+            `CREATE TABLE IF NOT EXISTS assistant (
+                id uuid NOT NULL DEFAULT uuid_generate_v4(),
+                "credential" varchar NOT NULL,
+                "details" text NOT NULL,
+                "iconSrc" varchar NULL,
+                "createdDate" timestamp NOT NULL DEFAULT now(),
+                "updatedDate" timestamp NOT NULL DEFAULT now(),
+                CONSTRAINT "PK_3c7cea7a044ac4c92764576cdbf" PRIMARY KEY (id)
+            );`
+        )
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE assistant`)
+    }
+}

packages/server/src/database/migrations/postgres/index.ts

@@ -6,6 +6,7 @@ import { ModifyTool1693997339912 } from './1693997339912-ModifyTool'
 import { AddApiConfig1694099183389 } from './1694099183389-AddApiConfig'
 import { AddAnalytic1694432361423 } from './1694432361423-AddAnalytic'
 import { AddChatHistory1694658756136 } from './1694658756136-AddChatHistory'
+import { AddAssistantEntity1699325775451 } from './1699325775451-AddAssistantEntity'
 
 export const postgresMigrations = [
     Init1693891895163,
@@ -15,5 +16,6 @@ export const postgresMigrations = [
     ModifyTool1693997339912,
     AddApiConfig1694099183389,
     AddAnalytic1694432361423,
-    AddChatHistory1694658756136
+    AddChatHistory1694658756136,
+    AddAssistantEntity1699325775451
 ]