Add File Service Interface and Document Model (#69)

* Add File Service Interface and Document Model

* Add File Interface

* Generate Document Model Migration

* Fix Core Config

* Add Document Service Draft

* Rename Document To Document Metadata

* Add Document Metadata Service

packages/ocular/core-config.js

@@ -16,7 +16,7 @@ switch (process.env.NODE_ENV) {
     ENV_FILE_NAME = ".env.dev";
     break;
   default:
-    ENV_FILE_NAME = ".env.";
+    ENV_FILE_NAME = ".env.dev";
     break;
 }
 

packages/ocular/package.json

@@ -30,8 +30,10 @@
   },
   "dependencies": {
     "@azure/search-documents": "^12.0.0",
+    "@medusajs/file-local": "^1.0.3",
     "@ocular/types": "*",
     "@ocular/utils": "*",
+    "@types/multer": "^1.4.11",
     "connect-redis": "^7.1.1",
     "cookie-parser": "^1.4.6",
     "core-js": "^3.36.1",

packages/ocular/src/loaders/plugins.ts

@@ -16,7 +16,7 @@ import {
 } from "../utils/format-registration-name"
 import {  aliasTo, asValue, asFunction , Lifetime } from "awilix"
 import { AbstractNotificationService, AbstractSearchService, OauthService } from "@ocular/types"
-import { AbstractDocumentProcesserService, AbstractLLMService } from "@ocular/types";
+import { AbstractDocumentProcesserService, AbstractLLMService, AbstractFileService } from "@ocular/types";
 import { AbstractVectorDBService } from "@ocular/types";
 
 type Options = {
@@ -275,6 +275,18 @@ export async function registerServices(
           // Register Service as vectorDBService for easy resolution.
           [`searchIndexService`]: aliasTo(name),
         })
+      } else if (AbstractFileService.isFileService(loaded.prototype)) {
+        // Add the service directly to the container in order to make simple
+        // resolution if we already know which file storage provider we need to use
+        container.register({
+          [name]: asFunction(
+            (cradle) => new loaded(cradle, pluginDetails.options),
+            {
+              lifetime: loaded.LIFE_TIME || Lifetime.SINGLETON,
+            }
+          ),
+          [`fileService`]: aliasTo(name),
+        })
       } else {
         container.register({
           [name]: asFunction(

packages/ocular/src/migrations/1715054351330-adddocumentmetadatamodel.ts

@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class Adddocumentmetadatamodel1715054351330 implements MigrationInterface {
+    name = 'Adddocumentmetadatamodel1715054351330'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`CREATE TYPE "public"."document_metadata_type_enum" AS ENUM('pdf', 'text', 'docx', 'html', 'md')`);
+        await queryRunner.query(`CREATE TYPE "public"."document_metadata_source_enum" AS ENUM('asana', 'confluence', 'github', 'gmail', 'google-drive', 'jira', 'notion', 'slack')`);
+        await queryRunner.query(`CREATE TABLE "document_metadata" ("id" character varying NOT NULL, "created_at" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updated_at" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "link" character varying NOT NULL, "title" character varying NOT NULL, "type" "public"."document_metadata_type_enum" NOT NULL, "source" "public"."document_metadata_source_enum" NOT NULL, "organisation_id" character varying NOT NULL, CONSTRAINT "UQ_b213a1ec4a3a720fed5cf55aa6d" UNIQUE ("link"), CONSTRAINT "PK_74b81e5979bc9e440e8d40f07e4" PRIMARY KEY ("id"))`);
+        await queryRunner.query(`ALTER TABLE "o_auth" ALTER COLUMN "last_sync" SET DEFAULT NULL`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "o_auth" ALTER COLUMN "last_sync" DROP DEFAULT`);
+        await queryRunner.query(`DROP TABLE "document_metadata"`);
+        await queryRunner.query(`DROP TYPE "public"."document_metadata_source_enum"`);
+        await queryRunner.query(`DROP TYPE "public"."document_metadata_type_enum"`);
+    }
+
+}

packages/ocular/src/models/document-metadata.ts

@@ -0,0 +1,38 @@
+import {
+  BeforeInsert,
+  Entity,
+  PrimaryGeneratedColumn,
+  Column,
+  ManyToOne,
+  JoinColumn,
+  OneToMany
+} from 'typeorm';
+import { AppNameDefinitions, BaseEntity, DocType } from "@ocular/types"
+import { Organisation } from './organisation';
+import { DbAwareColumn } from '@ocular/utils';
+
+@Entity()
+export class DocumentMetadata extends BaseEntity {
+  @Column({type: "varchar", nullable: false, unique: true})
+  link: string;
+
+  @Column({type: "varchar", nullable: false})
+  title: string;
+
+  @DbAwareColumn({
+    type: "enum",
+    enum: DocType,
+    nullable: false,
+  })
+  type: DocType
+
+  @DbAwareColumn({
+    type: "enum",
+    enum: AppNameDefinitions,
+    nullable: false,
+  })
+  source: AppNameDefinitions
+
+  @Column({type: "varchar", nullable: false})
+  organisation_id: string;
+}

packages/ocular/src/models/index.ts

@@ -1,6 +1,7 @@
 export * from "./app"
 export * from "./batch-job"
 export * from "./chat"
+export * from "./document-metadata"
 export * from "./message"
 export * from "./oauth"
 export * from "./organisation"

packages/ocular/src/repositories/document-metadata.ts

@@ -0,0 +1,5 @@
+import { DocumentMetadata } from "../models"
+import { dataSource } from "../loaders/database"
+
+export const DocumentMetadataRepository = dataSource.getRepository(DocumentMetadata)
+export default DocumentMetadataRepository

packages/ocular/src/repositories/index.ts

@@ -1,6 +1,7 @@
 export * from "./app"
 export * from "./batch-job"
 export * from "./chat"
+export * from "./document-metadata"
 export * from "./message"
 export * from "./oauth"
 export * from "./user"