diff --git a/chatapi.rest b/chatapi.rest new file mode 100644 index 000000000..5b5a168c8 --- /dev/null +++ b/chatapi.rest @@ -0,0 +1,30 @@ +POST http://localhost:5080/chatapi/v1/chat +Content-Type: application/json + +{ + "title" : "Hello", + "_experient_id": "63f6e4987c5f93004a3e3ca8", + "_dataset_id": "63f6e4947c5f93004a3e3ca7" +} + +### + +POST http://localhost:5080/chatapi/v1/chatlog/ +Content-Type: application/json + +{ + "_chat_id" : "63f6e54fc76ccd8396386d03", + "message" : "Hello there from cyberspace!", + "message_type" : "text", + "who" : "openai" +} + +### + +GET http://localhost:5080/chatapi/v1/chat + +### + +GET http://localhost:5080/chatapi/v1/chat/63f6e54fc76ccd8396386d03 + +### diff --git a/config/common.env b/config/common.env index 877fdde30..1eec1c76c 100644 --- a/config/common.env +++ b/config/common.env @@ -23,4 +23,7 @@ DT_MAX_DEPTH=6 DOCKER_CLIENT_TIMEOUT=120 COMPOSE_HTTP_TIMEOUT=120 +OPENAI_API_KEY=your_openai_api_key +OPENAI_ORG_ID=your_openai_org_id + STARTUP_DATASET_PATH=/appsrc/data/datasets/user diff --git a/lab/dbgoose.js b/lab/dbgoose.js new file mode 100644 index 000000000..027f4b33e --- /dev/null +++ b/lab/dbgoose.js @@ -0,0 +1,16 @@ +const mongoose = require('mongoose'); + +if (process.env.DBMONGO_HOST && process.env.DBMONGO_PORT) { + mongouri="mongodb://"+process.env.DBMONGO_HOST+":"+process.env.DBMONGO_PORT+"/FGLab"; +} else if (process.env.MONGODB_URI) { + mongouri=process.env.MONGODB_URI; +} else { + console.log("Error: No MongoDB instance specified"); + process.exit(1); +} +mongoose.connect(mongouri, { useNewUrlParser: true, useUnifiedTopology: true }) +const db = mongoose.connection; +db.on('error', (error) => console.error(error)); +db.once('open', () => console.log('Mongoose connected to Database')); + +module.exports = db; \ No newline at end of file diff --git a/lab/lab.js b/lab/lab.js index d6dfd10bb..b9722e0b2 100644 --- a/lab/lab.js +++ b/lab/lab.js @@ -51,7 +51,8 @@ var emitEvent = require("./socketServer").emitEvent; var generateFeaturesFromFileIdAsync = require("./pyutils").generateFeaturesFromFileIdAsync; var validateDatafileByFileIdAsync = require("./pyutils").validateDatafileByFileIdAsync; const assert = require("assert"); - +const openaiRouter = require('./routes/openai'); +const chatapiRouter = require('./routes/chatapi'); /*************** * Enums @@ -128,8 +129,10 @@ app.use(bodyParser.urlencoded({limit: "50mb", extended: true, parameterLimit:500 app.set('appPath', path.join(path.normalize(__dirname), 'webapp/dist')); app.use(express.static(app.get('appPath'))); +app.use('/openai/v1', openaiRouter); +app.use('/chatapi/v1', chatapiRouter); -/* API */ +/* Lab API */ // Registers webhooks app.post("/api/v1/webhooks", jsonParser, (req, res) => { diff --git a/lab/models/chat.js b/lab/models/chat.js new file mode 100644 index 000000000..e1ca4b0a3 --- /dev/null +++ b/lab/models/chat.js @@ -0,0 +1,21 @@ +const mongoose = require('mongoose'); + +const chatSchema = new mongoose.Schema({ + title: { + type: String, + required: true + }, + _dataset_id: { + type: mongoose.Schema.Types.ObjectId, + }, + _experiment_id: { + type: mongoose.Schema.Types.ObjectId, + }, + date: { + type: Date, + required: true, + default: Date.now + } +}); + +module.exports = mongoose.model('Chat', chatSchema); \ No newline at end of file diff --git a/lab/models/chatlog.js b/lab/models/chatlog.js new file mode 100644 index 000000000..0c754b941 --- /dev/null +++ b/lab/models/chatlog.js @@ -0,0 +1,27 @@ +const mongoose = require('mongoose'); + +const chatlogSchema = new mongoose.Schema({ + _chat_id: { + type: mongoose.Schema.Types.ObjectId, + required: true + }, + message: { + type: String, + }, + message_type: { + type: String, + }, + source_code: { + type: String, + }, + who: { + type: String, + }, + date: { + type: Date, + required: true, + default: Date.now + } +}); + +module.exports = mongoose.model('Chatlog', chatlogSchema); \ No newline at end of file diff --git a/lab/models/openaiconfig.js b/lab/models/openaiconfig.js new file mode 100644 index 000000000..4dbc221fb --- /dev/null +++ b/lab/models/openaiconfig.js @@ -0,0 +1,14 @@ +const mongoose = require('mongoose'); + +const openaiconfigSchema = new mongoose.Schema({ + org_id: { + type: String, + required: true + }, + api_key: { + type: String, + required: true + }, +}); + +module.exports = mongoose.model('OpenAIConfig', openaiconfigSchema); diff --git a/lab/models/openailog.js b/lab/models/openailog.js new file mode 100644 index 000000000..25b321f74 --- /dev/null +++ b/lab/models/openailog.js @@ -0,0 +1,20 @@ +const mongoose = require('mongoose'); + +const openailogSchema = new mongoose.Schema({ + request: { + type: mongoose.Schema.Types.Mixed, + required: true + }, + response: { + type: mongoose.Schema.Types.Mixed, + required: true + }, + requestDate: { + type: Date, + required: true, + default: Date.now + } +}); + + +module.exports = mongoose.model('OpenAILog', openailogSchema); \ No newline at end of file diff --git a/lab/openaiutils.js b/lab/openaiutils.js new file mode 100644 index 000000000..9a55584ae --- /dev/null +++ b/lab/openaiutils.js @@ -0,0 +1,56 @@ + +const OpenAILog = require('./models/openailog'); +const OpenAIConfig = require('./models/openaiconfig');7 +const db = require('./dbgoose').db; +const Chat = require('./models/chat'); + + +async function getConfigById(req, res, next) { + let config; + try { + config = await OpenAIConfig.findById(req.params.id); + if (config == null) { + return res.status(404).json({ message: 'Cannot find config' }); + } + } catch (err) { + return res.status(500).json({ message: err.message }); + } + + res.config = config; + next(); +} + +async function logOpenAIRequest(params, response) { + const openailog = new OpenAILog({ + request: params, + response: response + }); + try { + const newLog = await openailog.save(); + console.log('openai log saved'); + } catch (err) { + console.log('error: openai log not saved'); + } + // call our API to log the request to our server here. +} + +async function getChatById(req, res, next) { + let chat; + try { + chat = await Chat.findById(req.params.id); + if (chat == null) { + return res.status(404).json({ message: 'Cannot find chat' }); + } + } catch (err) { + return res.status(500).json({ message: err.message }); + } + + res.chat = chat; + next(); +} + +module.exports = { + getConfigById, + logOpenAIRequest, + getChatById +}; \ No newline at end of file diff --git a/lab/package.json b/lab/package.json index 505712ad3..a19be1634 100644 --- a/lab/package.json +++ b/lab/package.json @@ -28,8 +28,10 @@ "lodash": "^4.17.21", "mongodb": "^2.2.35", "mongoskin": "^2.1.0", + "mongoose": "^6.0.5", "morgan": "^1.10.0", "multer": "^1.4.2", + "openai": "^3.0.0", "request": "^2.88.2", "request-promise": "^4.2.5", "serve-favicon": "^2.3.0", diff --git a/lab/routes/chatapi.js b/lab/routes/chatapi.js new file mode 100644 index 000000000..115594d31 --- /dev/null +++ b/lab/routes/chatapi.js @@ -0,0 +1,207 @@ +const express = require('express'); +const router = express.Router(); +const Chat = require('../models/chat'); +const ChatLog = require('../models/chatlog'); +const db = require('../dbgoose').db; +const getChatById = require('../openaiutils').getChatById; + +/* +** Chat Logs API +*/ +// Create one chat log entry +router.post('/chatlog', async (req, res) => { + if (req.body._chat_id == null) { + return res.status(400).json({ message: 'Must provide a _chat_id' }); + } + if (req.body.message == null) { + return res.status(400).json({ message: 'Must provide a message' }); + } + if (req.body.message_type == null) { + return res.status(400).json({ message: 'Must provide a message_type' }); + } + + const chatlog = new ChatLog({ + _chat_id: req.body._chat_id, + message: req.body.message, + message_type: req.body.message_type, + source_code: req.body.source_code, + who: req.body.who, + }); + + try { + const newChatLog = await chatlog.save(); + res.status(201).json(newChatLog); + } catch (err) { + res.status(400).json({ message: err.message }); + } +}); + +// Update one chat log entry +router.patch('/chatlog/:id', getChatById, async (req, res) => { + // we should not allowed to update the _chat_id, as this would move this log to another chat + // if (req.body._chat_id != null) { + // res.chatlog._chat_id = req.body._chat_id; + // } + if (req.body.message != null) { + res.chatlog.message = req.body.message; + } + if (req.body.message_type != null) { + res.chatlog.message_type = req.body.message_type; + } + if (req.body.source_code != null) { + res.chatlog.source_code = req.body.source_code; + } + if (req.body.who != null) { + res.chatlog.who = req.body.who; + } + try { + const updatedChatLog = await res.chatlog.save(); + res.send(updatedChatLog); + } catch (err) { + res.status(400).json({ message: err.message }); + } +}); + +// On second thought, the GET chat/:id can return the chatlogs +// // Get all chat logs by chat_id from the chat collection +// // Test if there is no need to get a chat by id, I need to see if the +// // res.chat will be available in the response after res.send(chatlog) +// router.get('/chatlog/:chat_id', getChatById, async (req, res) => { +// try { +// const chatlog = await ChatLog.find({ _chat_id: res.chat._id }); +// res.send(chatlog); +// } catch (err) { +// res.status(500).json({ message: err.message }); +// } +// }); + + +/* +** Chat API +*/ +// Create one chat +router.post('/chat', async (req, res) => { + // a chat should always happen within the context of a dataset + if (req.body._dataset_id == null) { + return res.status(400).json({ message: 'Must provide a _dataset_id' }); + } + + if (req.body.title == null) { + return res.status(400).json({ message: 'Must provide a title' }); + } + + // we can prevent chats with a duplicat title by checking here + // but chatGPT currently allows duplicates. We can revisit this later. + + const chat = new Chat({ + title: req.body.title, + _dataset_id: req.body._dataset_id, + _experiment_id: req.body._experiment_id, + }); + + try { + const newChat = await chat.save(); + res.status(201).json(newChat); + } catch (err) { + res.status(400).json({ message: err.message }); + } +}); + +// Get all chats +// This endpoint will not return the chatlogs +router.get('/chat', async (req, res) => { + try { + const chat = await Chat.find(); + res.send(chat); + } catch (err) { + res.status(500).json({ message: err.message }); + } +}); + +// Get one chat +router.get('/chat/:id', getChatById, (req, res) => { + // get all chatlogs associated with this chat + ChatLog.find({ _chat_id: res.chat._id }, function (err, chatlogs) { + if (err) { + res.status(500).json({ message: err.message }); + } + res.send({ chat: res.chat, chatlogs: chatlogs }); + }); +}); + + +// Update one chat +router.patch('/chat/:id', getChatById, async (req, res) => { + if (req.body.title != null) { + res.chat.title = req.body.title; + } + if (req.body._dataset_id != null) { + res.chat._dataset_id = req.body._dataset_id; + } + if (req.body._experiment_id != null) { + res.chat._experiment_id = req.body._experiment_id; + } + try { + const updatedChat = await res.chat.save(); + res.send(updatedChat); + } catch (err) { + res.status(400).json({ message: err.message }); + } +}); + +// Delete one chat +router.delete('/chat/:id', getChatById, async (req, res) => { + try { + await res.chat.remove(); + // remove all chatlogs associated with this chat + await ChatLog.deleteMany({ _chat_id: res.chat._id }); + + res.send({ message: 'Deleted chat' }); + } catch (err) { + res.status(500).json({ message: err.message }); + } +}); + +// // Get one chat log entry by id +// async function getChatLogById(req, res, next) { +// let chatlog; +// try { +// chatlog = await ChatLog.findById(req.params.id); +// if (chatlog == null) { +// return res.status(404).json({ message: 'Cannot find chatlog' }); +// } +// } catch (err) { +// return res.status(500).json({ message: err.message }); +// } + +// console.log('getChatLogById:', chatlog); + +// res.chatlog = chatlog; +// next(); +// } + +// Update one chat log entry +router.patch('/chatlog/:id', getChatById, async (req, res) => { + // _chat_id should not be updated + if (req.body.message != null) { + res.chatlog.message = req.body.message; + } + if (req.body.message_type != null) { + res.chatlog.message_type = req.body.message_type; + } + if (req.body.source_code != null) { + res.chatlog.source_code = req.body.source_code; + } + if (req.body.who != null) { + res.chatlog.who = req.body.who; + } + try { + const updatedChatLog = await res.chatlog.save(); + res.send(updatedChatLog); + } catch (err) { + res.status(400).json({ message: err.message }); + } +}); + + +module.exports = router; \ No newline at end of file diff --git a/lab/routes/openai.js b/lab/routes/openai.js new file mode 100644 index 000000000..e7e32dd9a --- /dev/null +++ b/lab/routes/openai.js @@ -0,0 +1,141 @@ +const { Configuration, OpenAIApi } = require('openai'); +const express = require('express'); +const router = express.Router(); +const db = require('../dbgoose').db; +const getConfigById = require('../openaiutils').getConfigById; +const logOpenAIRequest = require('../openaiutils').logOpenAIRequest; +const OpenAIConfig = require('../models/openaiconfig'); +const configuration = new Configuration({ + // the organization is not required, so we need to decide if we require it from the user + // this may become useful for our logs. + // organization: process.env.OPENAI_ORG_ID, + apiKey: process.env.OPENAI_API_KEY, +}); +const openai = new OpenAIApi(configuration); + +/* +** OpenAI Config Settings API +*/ +// Create a config +router.post('/config', async (req, res) => { + // Check if config already exists + let existing; + try { + existing = await OpenAIConfig.find({}) + if (existing.length > 0) { + return res.status(400).json({ message: 'Config already exists' }); + } + } catch (err) { + return res.status(500).json({ message: err.message }); + } + + const config = new OpenAIConfig({ + org_id: req.body.org_id, + api_key: req.body.api_key + }); + try { + const newConfig = await config.save(); + res.status(201).json(newConfig); + } catch (err) { + res.status(400).json({ message: err.message }); + } +}); + +// Get all configs +router.get('/config', async (req, res) => { + try { + const config = await OpenAIConfig.find(); + res.send(config); + } catch (err) { + res.status(500).json({ message: err.message }); + } +}); + +// Get one config +router.get('/config/:id', getConfigById, (req, res) => { + res.send(res.config); +}); + +// Update one config +router.patch('/config/:id', getConfigById, async (req, res) => { + if (req.body.org_id != null) { + res.config.org_id = req.body.org_id; + } + if (req.body.api_key != null) { + res.config.api_key = req.body.api_key; + } + try { + const updatedConfig = await res.config.save(); + res.send(updatedConfig); + } catch (err) { + res.status(400).json({ message: err.message }); + } +}); + +// Delete one config +router.delete('/config/:id', getConfigById, async (req, res) => { + try { + await res.config.remove(); + res.send({ message: 'Deleted config' }); + } catch (err) { + res.status(500).json({ message: err.message }); + } +}); + +/* +** OpenAI API Requests +*/ +// get models +router.get('/models', async (req, res) => { + try { + let response = await openai.listModels(); + response = response.data; + console.log(response) + res.send(response); + } catch (err) { + res.status(500).json({ message: err.message }); + } +}); + +// get model by model name +router.get('/models/:model', async (req, res) => { + // Should I let the OpenAI api handle this? + // if (req.params.model == null) { + // return res.status(400).json({ message: 'No model provided' }); + // } + try { + let model = await openai.retrieveModel(req.params.model); + // model = removeCircularReference(model); + model = model.data; + res.send(model); + } catch (err) { + res.status(500).json({ message: err.message }); + } +}); + +// send a completion request +router.post('/completions', async (req, res) => { + let params = req.body; + let response = await openai.createCompletion(params); + response = response.data; + // chats should be logged by the client + // logChats(params, response); + logOpenAIRequest(req.body, response); + res.send(response); +}); + +// send an edit request +router.post('/edits', async (req, res) => { + let params = req.body; + + let response = await openai.createEdit(params); + response = response.data; + + // log the response in the openai collection in mongodb + console.log(response); + logOpenAIRequest(req.body, response); + + res.send(response); +}); + +module.exports = router; \ No newline at end of file diff --git a/openai.rest b/openai.rest new file mode 100644 index 000000000..5edd3164e --- /dev/null +++ b/openai.rest @@ -0,0 +1,38 @@ +GET http://localhost:5080/openai/v1/config + +### + +GET http://localhost:5080/openai/v1/config/63f6e6dcc76ccd8396386d18 + +### + +POST http://localhost:5080/openai/v1/config +Content-Type: application/json + +{ + "org_id" : "org_id2", + "api_key" : "api_key2" +} + +### + +GET http://localhost:5080/openai/v1/models + +### + +POST http://localhost:5080/openai/completions +Content-Type: application/json + +{ + "model" : "text-davinci-001", + "prompt" : "Test", + "max_tokens" : 7, + "temperature" : 0, + "top_p" : 1, + "n" : 1, + "stream" : false, + "logprobs" : null, + "stop" : "\n" +} + +### \ No newline at end of file