Added ingests of assets, files and fileObject #138

client/src/components/project/pam/ProjectViewer.vue

@@ -182,7 +182,7 @@
           </q-item>
         </div>
         <div class="col-12">
-          <FilesManager></FilesManager>
+          <FileManager></FileManager>
         </div>
       </div>
     </q-page>
@@ -194,13 +194,13 @@ import { date } from 'quasar'
 import { gql } from '@apollo/client'
 
 import CitationBuilder from '../../../components/CitationBuilder'
-import FilesManager from '../../../components/project/FilesManager'
+import FileManager from '../../../components/project/FileManager'
 
 export default {
   name: 'PageId',
   components: {
     CitationBuilder,
-    FilesManager
+    FileManager
   },
   data: () => ({
     projectIdFromRoute: null,

gql/insertAsset.gql

@@ -3,13 +3,15 @@ mutation insertAsset (
     $name: String!
     $assetType: asset_types_enum!
     $privacyType: privacy_types_enum!
+    $parentId: Int
 ) {
   insert_assets(
       objects: {
           assetType: $assetType, 
           createdById: $createdById, 
           name: $name, 
-          privacyType: $privacyType
+          privacyType: $privacyType,
+          parentId: $parentId
     }
     ) {
         returning {

server-nest/src/asset/asset.service.ts

@@ -16,10 +16,7 @@ export class AssetService {
     const { returning: assets } = await this.client.adminMutate(
       resolve(GQL_DIR, 'insertAsset.gql'),
       {
-        createdById: asset.createdById,
-        name: asset.name,
-        assetType: asset.assetType,
-        privacyType: asset.privacyType
+        ...asset
       }
     )
 

server-nest/src/dtos/asset.dto.ts

@@ -7,6 +7,7 @@ export class AssetDTO {
   assetType: AssetType
   privacyType: PrivacyType
   parentId: number
+  datetimeCreated: string
 
   constructor (asset: Partial<AssetDTO>) {
     Object.assign(this, asset)

server-nest/src/ingest/ingest.controller.ts

@@ -1,4 +1,5 @@
-import { Controller, Post, Request, Res } from '@nestjs/common'
+import { Controller, Post, Request, Res, UseInterceptors, UploadedFile } from '@nestjs/common'
+import { FileInterceptor } from '@nestjs/platform-express'
 import { IngestService } from './ingest.service'
 
 @Controller('ingest')
@@ -17,4 +18,16 @@ export class IngestController {
     console.log('Users ingest succeded!', result)
     res.status(200).send()
   }
+
+  @Post('projects')
+  @UseInterceptors(FileInterceptor('file'))
+  // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
+  async ingestProjects (@UploadedFile() file, @Request() req, @Res() res): Promise<void> {
+    const result = this.ingestService.ingestProjects()
+    console.log('Users ingest succeded!', result)
+    result.catch((error) => {
+      console.log(error)
+    })
+    res.status(200).send()
+  }
 }

server-nest/src/ingest/ingest.module.ts

@@ -1,11 +1,22 @@
 import { Module } from '@nestjs/common'
+import { MulterModule } from '@nestjs/platform-express'
+import { AssetModule } from 'src/asset/asset.module'
+import { FileModule } from 'src/file/file.module'
 import { KeycloakModule } from 'src/keycloak/keycloak.module'
 import { UserModule } from '../users/user.module'
 import { IngestController } from './ingest.controller'
 import { IngestService } from './ingest.service'
 
 @Module({
-  imports: [KeycloakModule, UserModule],
+  imports: [
+    MulterModule.register({
+      dest: '../data'
+    }),
+    KeycloakModule,
+    UserModule,
+    AssetModule,
+    FileModule
+  ],
   controllers: [IngestController],
   providers: [IngestService]
 })

server-nest/src/ingest/ingest.service.ts

@@ -2,20 +2,24 @@ import { Injectable } from '@nestjs/common'
 import { KeycloakService } from '../keycloak/keycloak.service'
 import { UserService } from '../users/user.service'
 import { UserDTO } from '../dtos/user.dto'
+import { AssetDTO } from '../dtos/asset.dto'
+import { AssetService } from 'src/asset/asset.service'
+import { readFileSync } from 'fs'
+import { resolve } from 'path'
+import { FileService } from 'src/file/file.service'
+import { FileObjectDTO } from 'src/dtos/fileobject.dto'
+import { FileDTO } from 'src/dtos/file.dto'
 
 @Injectable()
 export class IngestService {
   constructor (
     private readonly keycloakService: KeycloakService,
-    private readonly userService: UserService
+    private readonly userService: UserService,
+    private readonly assetService: AssetService,
+    private readonly fileService: FileService
   ) {}
 
   async ingestUsers (users: Array<Record<string, string>>): Promise<boolean> {
-    // "givenName": "Karen",
-    // "familyName": "Adolph",
-    // "displayFullName": "Karen Adolph",
-    // "emailPrimary": "karen.adolph@nyu.edu",
-    // "authServerId": "7768e08c-d75a-4771-877f-8d26a842c773"
     for (const user of users) {
       console.log('Inegst user', user)
 
@@ -45,4 +49,77 @@ export class IngestService {
     }
     return true
   }
+
+  async ingestProjects (): Promise<boolean> {
+    const data = readFileSync(resolve('../data/projects_ingest.json'), 'utf8')
+    const projects = JSON.parse(data)
+    if (projects == null) return false
+
+    for (const project of projects) {
+      const {
+        name,
+        createdBy,
+        privacyType,
+        assetType,
+        folders
+      } = project
+
+      //   Find user id from email
+      const user = await this.userService.findByEmail(createdBy)
+      if (user == null) return
+
+      //   Create a project asset
+      const projectAsset = await this.assetService.insertAsset(new AssetDTO({
+        createdById: user.id,
+        name: name,
+        assetType: assetType,
+        privacyType: privacyType
+      }))
+
+      for (const folder of folders) {
+        const folderAsset = await this.assetService.insertAsset(new AssetDTO({
+          createdById: user.id,
+          name: folder.name,
+          assetType: folder.assetType,
+          privacyType: folder.privacyType,
+          parentId: projectAsset.id
+        }))
+
+        const { files } = folder
+        if (files == null) continue
+
+        for (const fi of files) {
+          const fileAsset = await this.assetService.insertAsset(new AssetDTO({
+            createdById: user.id,
+            name: fi.name,
+            assetType: fi.assetType,
+            privacyType: fi.privacyType,
+            parentId: folderAsset.id
+          }))
+
+          const { file } = fi
+
+          const fileobjectsId = await this.fileService
+            .insertFileObject(new FileObjectDTO({
+              location: 's3://minio-1.nyu.edu/cas',
+              size: file.fileobjects.size,
+              sha1: file.fileobjects.sha1,
+              sha256: file.fileobjects.sha256,
+              md5: file.fileobjects.md5
+            }))
+
+          await this.fileService
+            .insertFile(new FileDTO({
+              name: file.name,
+              assetId: fileAsset.id,
+              fileFormatId: file.fileFormatId.length === 0 ? 'mp4' : file.fileFormatId,
+              uploadedById: user.id,
+              fileobjectId: fileobjectsId
+            // createdDateTime: new Date().toISOString()
+            }))
+        }
+      }
+    }
+    return true
+  }
 }