Skip to content

Commit

Permalink
Comment out currently unavailable methods
Browse files Browse the repository at this point in the history
  • Loading branch information
@miya
miya committed Jan 10, 2025
1 parent 2df37fd commit b18d188
Show file tree
Hide file tree
Showing 6 changed files with 91 additions and 93 deletions.
2 changes: 1 addition & 1 deletion apps/app/src/features/openai/server/routes/rebuild-vector-store.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ export const rebuildVectorStoreHandlersFactory: RebuildVectorStoreFactory = (cro

try {
const openaiService = getOpenaiService();
await openaiService?.rebuildVectorStoreAll();
// await openaiService?.rebuildVectorStoreAll();
return res.apiv3({});

}
Expand Down
6 changes: 3 additions & 3 deletions apps/app/src/features/openai/server/routes/thread.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@ export const createThreadHandlersFactory: CreateThreadFactory = (crowi) => {
try {
const openaiService = getOpenaiService();
const filterdThreadId = req.body.threadId != null ? filterXSS(req.body.threadId) : undefined;
const vectorStore = await openaiService?.getOrCreateVectorStoreForPublicScope();
const thread = await openaiService?.getOrCreateThread(req.user._id, vectorStore?.vectorStoreId, filterdThreadId);
return res.apiv3({ thread });
// const vectorStore = await openaiService?.getOrCreateVectorStoreForPublicScope();
// const thread = await openaiService?.getOrCreateThread(req.user._id, vectorStore?.vectorStoreId, filterdThreadId);
return res.apiv3({ });
}
catch (err) {
logger.error(err);
Expand Down
162 changes: 80 additions & 82 deletions apps/app/src/features/openai/server/services/openai.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import type OpenAI from 'openai';
import { toFile } from 'openai';

import ThreadRelationModel from '~/features/openai/server/models/thread-relation';
import VectorStoreModel, { VectorStoreScopeType, type VectorStoreDocument } from '~/features/openai/server/models/vector-store';
import VectorStoreModel, { type VectorStoreDocument } from '~/features/openai/server/models/vector-store';
import VectorStoreFileRelationModel, {
type VectorStoreFileRelation,
prepareVectorStoreFileRelations,
Expand All @@ -35,20 +35,20 @@ const BATCH_SIZE = 100;

const logger = loggerFactory('growi:service:openai');

let isVectorStoreForPublicScopeExist = false;
// const isVectorStoreForPublicScopeExist = false;

type VectorStoreFileRelationsMap = Map<string, VectorStoreFileRelation>

export interface IOpenaiService {
getOrCreateThread(userId: string, vectorStoreId?: string, threadId?: string): Promise<OpenAI.Beta.Threads.Thread | undefined>;
getOrCreateVectorStoreForPublicScope(): Promise<VectorStoreDocument>;
// getOrCreateVectorStoreForPublicScope(): Promise<VectorStoreDocument>;
deleteExpiredThreads(limit: number, apiCallInterval: number): Promise<void>; // for CronJob
deleteObsolatedVectorStoreRelations(): Promise<void> // for CronJob
createVectorStoreFile(vectorStore: VectorStoreDocument, pages: PageDocument[]): Promise<void>;
createVectorStoreFile(vectorStoreRelation: VectorStoreDocument, pages: PageDocument[]): Promise<void>;
deleteVectorStoreFile(vectorStoreRelationId: Types.ObjectId, pageId: Types.ObjectId): Promise<void>;
deleteObsoleteVectorStoreFile(limit: number, apiCallInterval: number): Promise<void>; // for CronJob
rebuildVectorStoreAll(): Promise<void>;
rebuildVectorStore(page: HydratedDocument<PageDocument>): Promise<void>;
// rebuildVectorStoreAll(): Promise<void>;
// rebuildVectorStore(page: HydratedDocument<PageDocument>): Promise<void>;
createAiAssistant(data: Omit<AiAssistant, 'vectorStore'>): Promise<AiAssistantDocument>;
}
class OpenaiService implements IOpenaiService {
Expand Down Expand Up @@ -115,44 +115,43 @@ class OpenaiService implements IOpenaiService {
await ThreadRelationModel.deleteMany({ threadId: { $in: deletedThreadIds } });
}

public async getOrCreateVectorStoreForPublicScope(): Promise<VectorStoreDocument> {
const vectorStoreDocument: VectorStoreDocument | null = await VectorStoreModel.findOne({ scopeType: VectorStoreScopeType.PUBLIC, isDeleted: false });
// public async getOrCreateVectorStoreForPublicScope(): Promise<VectorStoreDocument> {
// const vectorStoreDocument: VectorStoreDocument | null = await VectorStoreModel.findOne({ scopeType: VectorStoreScopeType.PUBLIC, isDeleted: false });

if (vectorStoreDocument != null && isVectorStoreForPublicScopeExist) {
return vectorStoreDocument;
}
// if (vectorStoreDocument != null && isVectorStoreForPublicScopeExist) {
// return vectorStoreDocument;
// }

if (vectorStoreDocument != null && !isVectorStoreForPublicScopeExist) {
try {
// Check if vector store entity exists
// If the vector store entity does not exist, the vector store document is deleted
await this.client.retrieveVectorStore(vectorStoreDocument.vectorStoreId);
isVectorStoreForPublicScopeExist = true;
return vectorStoreDocument;
}
catch (err) {
await oepnaiApiErrorHandler(err, { notFoundError: vectorStoreDocument.markAsDeleted });
throw new Error(err);
}
}
// if (vectorStoreDocument != null && !isVectorStoreForPublicScopeExist) {
// try {
// // Check if vector store entity exists
// // If the vector store entity does not exist, the vector store document is deleted
// await this.client.retrieveVectorStore(vectorStoreDocument.vectorStoreId);
// isVectorStoreForPublicScopeExist = true;
// return vectorStoreDocument;
// }
// catch (err) {
// await oepnaiApiErrorHandler(err, { notFoundError: vectorStoreDocument.markAsDeleted });
// throw new Error(err);
// }
// }

const newVectorStore = await this.client.createVectorStore(VectorStoreScopeType.PUBLIC);
const newVectorStoreDocument = await VectorStoreModel.create({
vectorStoreId: newVectorStore.id,
scopeType: VectorStoreScopeType.PUBLIC,
}) as VectorStoreDocument;
// const newVectorStore = await this.client.createVectorStore(VectorStoreScopeType.PUBLIC);
// const newVectorStoreDocument = await VectorStoreModel.create({
// vectorStoreId: newVectorStore.id,
// scopeType: VectorStoreScopeType.PUBLIC,
// }) as VectorStoreDocument;

isVectorStoreForPublicScopeExist = true;
// isVectorStoreForPublicScopeExist = true;

return newVectorStoreDocument;
}
// return newVectorStoreDocument;
// }

private async createVectorStore(): Promise<VectorStoreDocument> {
try {
const newVectorStore = await this.client.createVectorStore(VectorStoreScopeType.PUBLIC); // TODO: fix argument
const newVectorStore = await this.client.createVectorStore();

const newVectorStoreDocument = await VectorStoreModel.create({
scopeType: VectorStoreScopeType.PUBLIC,
vectorStoreId: newVectorStore.id,
}) as VectorStoreDocument;

Expand Down Expand Up @@ -185,37 +184,37 @@ class OpenaiService implements IOpenaiService {
return uploadedFile;
}

private async deleteVectorStore(vectorStoreScopeType: VectorStoreScopeType): Promise<void> {
const vectorStoreDocument: VectorStoreDocument | null = await VectorStoreModel.findOne({ scopeType: vectorStoreScopeType, isDeleted: false });
if (vectorStoreDocument == null) {
return;
}
// private async deleteVectorStore(vectorStoreScopeType: VectorStoreScopeType): Promise<void> {
// const vectorStoreDocument: VectorStoreDocument | null = await VectorStoreModel.findOne({ scopeType: vectorStoreScopeType, isDeleted: false });
// if (vectorStoreDocument == null) {
// return;
// }

try {
await this.client.deleteVectorStore(vectorStoreDocument.vectorStoreId);
await vectorStoreDocument.markAsDeleted();
}
catch (err) {
await oepnaiApiErrorHandler(err, { notFoundError: vectorStoreDocument.markAsDeleted });
throw new Error(err);
}
}
// try {
// await this.client.deleteVectorStore(vectorStoreDocument.vectorStoreId);
// await vectorStoreDocument.markAsDeleted();
// }
// catch (err) {
// await oepnaiApiErrorHandler(err, { notFoundError: vectorStoreDocument.markAsDeleted });
// throw new Error(err);
// }
// }

async createVectorStoreFile(vectorStore: VectorStoreDocument, pages: Array<HydratedDocument<PageDocument>>): Promise<void> {
async createVectorStoreFile(vectorStoreRelation: VectorStoreDocument, pages: Array<HydratedDocument<PageDocument>>): Promise<void> {
// const vectorStore = await this.getOrCreateVectorStoreForPublicScope();
const vectorStoreFileRelationsMap: VectorStoreFileRelationsMap = new Map();
const processUploadFile = async(page: HydratedDocument<PageDocument>) => {
if (page._id != null && page.grant === PageGrant.GRANT_PUBLIC && page.revision != null) {
if (isPopulated(page.revision) && page.revision.body.length > 0) {
const uploadedFile = await this.uploadFile(page._id, page.path, page.revision.body);
prepareVectorStoreFileRelations(vectorStore._id, page._id, uploadedFile.id, vectorStoreFileRelationsMap);
prepareVectorStoreFileRelations(vectorStoreRelation._id, page._id, uploadedFile.id, vectorStoreFileRelationsMap);
return;
}

const pagePopulatedToShowRevision = await page.populateDataToShowRevision();
if (pagePopulatedToShowRevision.revision != null && pagePopulatedToShowRevision.revision.body.length > 0) {
const uploadedFile = await this.uploadFile(page._id, page.path, pagePopulatedToShowRevision.revision.body);
prepareVectorStoreFileRelations(vectorStore._id, page._id, uploadedFile.id, vectorStoreFileRelationsMap);
prepareVectorStoreFileRelations(vectorStoreRelation._id, page._id, uploadedFile.id, vectorStoreFileRelationsMap);
}
}
};
Expand Down Expand Up @@ -246,7 +245,7 @@ class OpenaiService implements IOpenaiService {
await VectorStoreFileRelationModel.upsertVectorStoreFileRelations(vectorStoreFileRelations);

// Create vector store file
const createVectorStoreFileBatchResponse = await this.client.createVectorStoreFileBatch(vectorStore.vectorStoreId, uploadedFileIds);
const createVectorStoreFileBatchResponse = await this.client.createVectorStoreFileBatch(vectorStoreRelation.vectorStoreId, uploadedFileIds);
logger.debug('Create vector store file', createVectorStoreFileBatchResponse);

// Set isAttachedToVectorStore: true when the uploaded file is attached to VectorStore
Expand All @@ -257,7 +256,7 @@ class OpenaiService implements IOpenaiService {

// Delete all uploaded files if createVectorStoreFileBatch fails
for await (const pageId of pageIds) {
await this.deleteVectorStoreFile(vectorStore._id, pageId);
await this.deleteVectorStoreFile(vectorStoreRelation._id, pageId);
}
}

Expand Down Expand Up @@ -349,32 +348,32 @@ class OpenaiService implements IOpenaiService {
}
}

async rebuildVectorStoreAll() {
await this.deleteVectorStore(VectorStoreScopeType.PUBLIC);

// Create all public pages VectorStoreFile
const Page = mongoose.model<HydratedDocument<PageDocument>, PageModel>('Page');
const pagesStream = Page.find({ grant: PageGrant.GRANT_PUBLIC }).populate('revision').cursor({ batch_size: BATCH_SIZE });
const batchStrem = createBatchStream(BATCH_SIZE);

const createVectorStoreFile = this.createVectorStoreFile.bind(this);
const createVectorStoreFileStream = new Transform({
objectMode: true,
async transform(chunk: HydratedDocument<PageDocument>[], encoding, callback) {
await createVectorStoreFile(chunk);
this.push(chunk);
callback();
},
});

await pipeline(pagesStream, batchStrem, createVectorStoreFileStream);
}
// async rebuildVectorStoreAll() {
// await this.deleteVectorStore(VectorStoreScopeType.PUBLIC);

// // Create all public pages VectorStoreFile
// const Page = mongoose.model<HydratedDocument<PageDocument>, PageModel>('Page');
// const pagesStream = Page.find({ grant: PageGrant.GRANT_PUBLIC }).populate('revision').cursor({ batch_size: BATCH_SIZE });
// const batchStrem = createBatchStream(BATCH_SIZE);

// const createVectorStoreFile = this.createVectorStoreFile.bind(this);
// const createVectorStoreFileStream = new Transform({
// objectMode: true,
// async transform(chunk: HydratedDocument<PageDocument>[], encoding, callback) {
// await createVectorStoreFile(chunk);
// this.push(chunk);
// callback();
// },
// });

// await pipeline(pagesStream, batchStrem, createVectorStoreFileStream);
// }

async rebuildVectorStore(page: HydratedDocument<PageDocument>) {
const vectorStore = await this.getOrCreateVectorStoreForPublicScope();
await this.deleteVectorStoreFile(vectorStore._id, page._id);
await this.createVectorStoreFile([page]);
}
// async rebuildVectorStore(page: HydratedDocument<PageDocument>) {
// const vectorStore = await this.getOrCreateVectorStoreForPublicScope();
// await this.deleteVectorStoreFile(vectorStore._id, page._id);
// await this.createVectorStoreFile([page]);
// }

async createAiAssistant(data: Omit<AiAssistant, 'vectorStore'>): Promise<AiAssistantDocument> {
// 1. Get pages stream based on path patterns
Expand All @@ -396,14 +395,13 @@ class OpenaiService implements IOpenaiService {
.cursor({ batchSize: BATCH_SIZE });
const batchStream = createBatchStream(BATCH_SIZE);

const vectorStore = await this.createVectorStore();

const vectorStoreRelation = await this.createVectorStore();
const createVectorStoreFile = this.createVectorStoreFile.bind(this);
const createVectorStoreFileStream = new Transform({
objectMode: true,
async transform(chunk: HydratedDocument<PageDocument>[], encoding, callback) {
try {
await createVectorStoreFile(vectorStore, chunk);
await createVectorStoreFile(vectorStoreRelation, chunk);
this.push(chunk);
callback();
}
Expand All @@ -418,7 +416,7 @@ class OpenaiService implements IOpenaiService {

// 4. Create AI Assistant with vector store (TODO)
const aiAssistant = await AiAssistantModel.create({
...data, vectorStore,
...data, vectorStore: vectorStoreRelation,
});

return aiAssistant;
Expand Down
2 changes: 1 addition & 1 deletion apps/app/src/server/routes/apiv3/page/create-page.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ export const createPageHandlersFactory: CreatePageHandlersFactory = (crowi) => {
const { getOpenaiService } = await import('~/features/openai/server/services/openai');
try {
const openaiService = getOpenaiService();
await openaiService?.rebuildVectorStore(createdPage);
// await openaiService?.rebuildVectorStore(createdPage);
}
catch (err) {
logger.error('Rebuild vector store failed', err);
Expand Down
2 changes: 1 addition & 1 deletion apps/app/src/server/routes/apiv3/page/update-page.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ export const updatePageHandlersFactory: UpdatePageHandlersFactory = (crowi) => {
const { getOpenaiService } = await import('~/features/openai/server/services/openai');
try {
const openaiService = getOpenaiService();
await openaiService?.rebuildVectorStore(updatedPage);
// await openaiService?.rebuildVectorStore(updatedPage);
}
catch (err) {
logger.error('Rebuild vector store failed', err);
Expand Down
10 changes: 5 additions & 5 deletions apps/app/src/server/service/page/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1175,7 +1175,7 @@ class PageService implements IPageService {

// Do not await because communication with OpenAI takes time
const openaiService = getOpenaiService();
openaiService?.createVectorStoreFile([duplicatedTarget]);
// openaiService?.createVectorStoreFile([duplicatedTarget]);
}
}
this.pageEvent.emit('duplicate', page, user);
Expand Down Expand Up @@ -1416,7 +1416,7 @@ class PageService implements IPageService {

// Do not await because communication with OpenAI takes time
const openaiService = getOpenaiService();
openaiService?.createVectorStoreFile(duplicatedPagesWithPopulatedToShowRevison);
// openaiService?.createVectorStoreFile(duplicatedPagesWithPopulatedToShowRevison);
}
}

Expand Down Expand Up @@ -1900,9 +1900,9 @@ class PageService implements IPageService {

const openaiService = getOpenaiService();
if (openaiService != null) {
const vectorStore = await openaiService.getOrCreateVectorStoreForPublicScope();
const deleteVectorStoreFilePromises = pageIds.map(pageId => openaiService.deleteVectorStoreFile(vectorStore._id, pageId));
await Promise.allSettled(deleteVectorStoreFilePromises);
// const vectorStore = await openaiService.getOrCreateVectorStoreForPublicScope();
// const deleteVectorStoreFilePromises = pageIds.map(pageId => openaiService.deleteVectorStoreFile(vectorStore._id, pageId));
// await Promise.allSettled(deleteVectorStoreFilePromises);
}
}
}
Expand Down

0 comments on commit b18d188

Please sign in to comment.