Skip to content

Commit

Permalink
Add files via upload
Browse files Browse the repository at this point in the history
  • Loading branch information
@dspasyuk
dspasyuk authored Dec 21, 2024
1 parent 86bd62c commit 0749cb1
Show file tree
Hide file tree
Showing 8 changed files with 516 additions and 242 deletions.
36 changes: 31 additions & 5 deletions config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,41 @@ const Hash = new hash();

const config = {};

config.AI = {llamacpp:true, groq:false}

config.modelrepo = "bartowski/Qwen2.5-7B-Instruct-GGUF";
config.modeldirectory = path.resolve('./models');
config.modelname = "Qwen2.5-7B-Instruct-Q4_0.gguf";

// Model Setting //Llama.cpp settings
config.systemPrompt = fs.readFileSync('./Alice.txt', 'utf8');
// config.systemPrompt= fs.readFileSync('Science.txt', 'utf8');

config.params = {

config.groqParameters = {
data:{
messages: [{
"role": "system",
"content": `'${config.systemPrompt}'`
},{
"role": "user",
"content": "",
}],
model: "llama-3.3-70b-versatile",
temperature: 0.3,
max_tokens: 8*1024,
top_p: 1,
stream: false,
stop: null},
APIkey: process.env.GROQ_API_KEY || ""
}

// create project, get api key and add Custom Search API to the project at https://console.cloud.google.com/
// create programable search engine, get ID https://programmablesearchengine.google.com

config.google = {
APIkey: process.env.GOOGLE_API_KEY || "",
SearchEngineID: process.env.GOOGLE_SEARCH_ENGINE_ID || "",
}

config.llamaParams = {
"--model": path.join(config.modeldirectory, config.modelname),
"--n-gpu-layers": 33, // remove if using CPU
"-cnv": "",
Expand Down Expand Up @@ -100,4 +126,4 @@ Answer the following questions:
8. Create a bouncing ball animation as all in one HTML/JS/CSS page.
`;

export default config;
export default config;
11 changes: 8 additions & 3 deletions package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "llama.cui",
"version": "0.3.40",
"version": "0.3.30",
"main": "server.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
Expand All @@ -12,15 +12,20 @@
"description": "llama.cui chat UI for llama.cpp",
"dependencies": {
"@xenova/transformers": "^2.17.2",
"any-text": "^1.2.0",
"axios": "^1.6.2",
"bcrypt": "^5.1.1",
"dompurify": "^3.2.3",
"ejs": "^3.1.9",
"express": "^4.19.2",
"express-session": "^1.17.3",
"jsdom": "^25.0.1",
"jszip": "^3.10.1",
"memorystore": "^1.6.7",
"pdf-text-reader": "^5.1.0",
"progress": "^2.0.3",
"socket.io": "^4.7.2",
"socket.io-client": "^4.7.2"
"socket.io-client": "^4.7.2",
"word-extractor": "^1.0.4",
"xml2js": "^0.6.2"
}
}
146 changes: 94 additions & 52 deletions server.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,13 @@ import cors from 'cors';
import path from 'path';
import { fileURLToPath } from 'url';
import vdb from './src/db.js';
import DDG from './src/ddg.js';
import GOOG from './src/goo.js';
import fs from 'fs';
import downloadModel from './src/modeldownloader.js';
import session from 'express-session';
import MemoryStoreModule from 'memorystore';
import hash from "./src/hash.js";
import axios from "axios";
const Hash = new hash();
import config from './config.js';

Expand All @@ -32,10 +33,10 @@ const version = 0.331; //changed public and server and config
function ser() {}

ser.modelinit = async function () {
if (fs.existsSync(config.params["--model"])) {
if (fs.existsSync(config.llamaParams["--model"])) {
console.log("Model exists");
} else {
console.log("Downloading the model", config.params["--model"]);
console.log("Downloading the model", config.llamaParams["--model"]);
console.log("Model repo: " + config.modelrepo, config.modelname, config.modeldirectory);
await downloadModel(
config.modelrepo,
Expand All @@ -46,14 +47,12 @@ ser.modelinit = async function () {
};

ser.init = function (error) {
console.log(
config.llamacpp + " " + Object.entries(config.params).flat().join(" ")
);
this.terminationtoken = "\n\n>";
this.connectedClients = new Map();
this.socketId = null;
this.messageQueue = []; // Queue to store messages from clients
this.isProcessing = false; // Flag to track if a message is being processed
this.runLLamaChild();
if (config.AI.llamacpp) this.runLLamaChild();
this.piper_client_enabled = true;
if (config.piper.enabled) {
this.fullmessage = "";
Expand Down Expand Up @@ -118,11 +117,15 @@ ser.init = function (error) {

this.app.post("/stopper", async (request, response) => {
console.log("STOPPING");
ser.llamachild.kill("SIGINT");
this.messageQueue.splice(0, 1);
this.isProcessing = false;
this.processMessageQueue();
response.send({ message: "stopped" });
if(config.AI.llamacpp){
ser.llamachild.kill("SIGINT");
this.messageQueue.splice(0, 1);
this.isProcessing = false;
this.processMessageQueue();
response.send({ message: "stopped" });
}else{
response.send({ message: "stopped" });
}
});

this.app.get("/login", (req, res) => {
Expand Down Expand Up @@ -199,7 +202,10 @@ ser.isValidSession = function (sessionID) {
};

ser.runLLamaChild = function () {
var configParams = Object.entries(config.params).flat();
console.log(
config.llamacpp + " " + Object.entries(config.llamaParams).flat().join(" ")
);
var configParams = Object.entries(config.llamaParams).flat();
this.llamachild = spawn(
config.llamacpp,
configParams.filter((item) => item !== ""),
Expand All @@ -222,6 +228,58 @@ ser.runLLamaChild = function () {
});
};

ser.runGroq = function (input) {
if(input.length!=0){
config.groqParameters.data.messages[1].content = input;
axios.post('https://api.groq.com/openai/v1/chat/completions',
JSON.stringify(config.groqParameters.data),
{
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${config.groqParameters.APIkey}`,
}
}).then(response => {
// console.log(JSON.stringify(response.data));
this.handleGroq(response.data.choices[0].message.content+this.terminationtoken);
}).catch(error => {
console.error(JSON.stringify(error));
});
}
}

ser.handleGroq = function (msg) {
const output = config.outputFilter(msg);
this.io.to(this.socketId).emit("output", output);
this.runPiper(output);
if (output) {
clearTimeout(this.streamTimeout);
}
this.messageQueue.splice(0, 1);
this.isProcessing = false;
this.processMessageQueue();
};


ser.handleLlama = function (msg) {
this.buffer += msg.toString("utf-8");
let lastSpaceIndex = this.buffer.lastIndexOf(" ");
if (lastSpaceIndex !== -1) {
let output = this.buffer.substring(0, lastSpaceIndex);
this.buffer = this.buffer.substring(lastSpaceIndex + 1);
output = config.outputFilter(output);
if (output) {
clearTimeout(this.streamTimeout);
}
this.io.to(this.socketId).emit("output", output);
this.runPiper(output);
if (output.includes(this.terminationtoken)) {
this.messageQueue.splice(0, 1);
this.isProcessing = false;
this.processMessageQueue();
}
}
};

ser.loggedIn = function (req, res, next) {
if (!config.login) {
req.session.loggedin = true;
Expand Down Expand Up @@ -270,17 +328,10 @@ ser.open = function () {
exec(start + ' ' + url);
};

ser.webseach = async function(input){
const ddg = new DDG({limit:4});
var ggsearch = [];
for await (const result of ddg.text(input)) {
if(result && result.content!=null){
// console.log(result);
ggsearch.push(result);
}
}

return ggsearch;
ser.webseach = async function(query){
const goog = new GOOG();
const results = await goog.searchGoogle(query, config.google.APIkey, config.google.SearchEngineID, 4)
return results;
}


Expand All @@ -306,28 +357,7 @@ ser.runPiper = function (output) {
}
};

ser.handleLlama = function (msg) {
this.buffer += msg.toString("utf-8");
let lastSpaceIndex = this.buffer.lastIndexOf(" ");
if (lastSpaceIndex !== -1) {
let output = this.buffer.substring(0, lastSpaceIndex);
this.buffer = this.buffer.substring(lastSpaceIndex + 1);
// output = parseOutput(output);
output = config.outputFilter(output);
// console.log(JSON.stringify(output));
if (output) {
clearTimeout(this.streamTimeout);
}
this.io.to(this.socketId).emit("output", output);
this.runPiper(output);
if (output.includes("\n\n>")) {
// console.log("Stopped");
this.messageQueue.splice(0, 1);
this.isProcessing = false;
this.processMessageQueue();
}
}
};


ser.processMessageQueue = function () {
if (this.messageQueue.length === 0) {
Expand All @@ -340,10 +370,19 @@ ser.processMessageQueue = function () {
this.socketId = socketId;
this.piper_client_enabled = piper;
// Send the message to the child process
this.llamachild.stdin.cork();
this.llamachild.stdin.write(`${input}`);
this.llamachild.stdin.write("\n");
this.llamachild.stdin.uncork();
if (config.AI.llamacpp){ ;
this.llamachild.stdin.cork();
this.llamachild.stdin.write(`${input}`);
this.llamachild.stdin.write("\n");
this.llamachild.stdin.uncork();
} else if(config.AI.groq) {
if (config.groqParameters.APIkey.length > 0) {
this.runGroq(input);
} else {
this.io.to(this.socketId).emit("output", 'Groq API key is missing or incorrect. Go to https://console.groq.com/keys to get your API key. Set your API key in the config.js file or using environment variables. On Unix/Linux/MacOS: export GROQ_API_KEY="your_api_key" On Windows: set GROQ_API_KEY="your_api_key"');
}
this.runGroq(input);
}
};

ser.handleTimeout = function () {
Expand Down Expand Up @@ -392,7 +431,7 @@ ser.handleSocketConnection = async function (socket) {
if (data.embedding.db) {
embedobj = embedobj.concat(await vdb.init(input));
}
if(config.embedding.WebSearch && data.embedding.web && input.length < 100){
if(config.embedding.WebSearch && data.embedding.web && input.length < 200){
const searchRes = await ser.webseach(input);
embedobj = embedobj.concat(searchRes);
}
Expand Down Expand Up @@ -452,7 +491,10 @@ ser.start = function () {


async function run() {
await ser.modelinit();
if(config.AI.llamacpp){
await ser.modelinit();
}

ser.init();
}
run();
Expand Down
14 changes: 8 additions & 6 deletions src/db.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,23 @@ import Hive from "./hive.js";
import path from "path";
import config from "../config.js";
import { exec } from "child_process";


import { fileURLToPath } from "url";
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(path.dirname(__filename)); // remove
console.log(__dirname);
function vdb() {}

vdb.init = async function (
query = "Human Factors Workscope",
) {
vdb.pipeline;
this.dbFile = "./db/Documents/db.json";
this.dbFile = path.join(__dirname, "/db/Documents/db.json");
this.index = {};
this.useLlamaEmbedding = false;
this.dataChannel = new Map();
this.dataChannel.set("Documents", {
datastream: "Documents",
datafolder: "./docs",
datafolder: path.join(__dirname, "docs"),
slice: 1024,
vectordb: "Documents.js"
});
Expand All @@ -34,8 +36,8 @@ vdb.init = async function (
return await vdb.query(query);
};

vdb.initVectorDB = async function (type = "Documents") {
await Hive.init(type, this.dbFile, this.dataChannel.get(type).datafolder);
vdb.initVectorDB = async function (type = "text") {
await Hive.init({type:type, pathToDB:this.dbFile, pathToDocs:this.dataChannel.get("Documents").datafolder});
};

vdb.query = async function (query, database = "Documents") {
Expand Down
Loading

0 comments on commit 0749cb1

Please sign in to comment.