From 7f502fa71f9a689a4c2f5561657ac3d11dfd854a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daute=20Rodr=C3=ADguez=20Rodr=C3=ADguez?= Date: Wed, 16 Feb 2022 12:16:00 +0000 Subject: [PATCH 1/3] fix: resourceId for newly created imports is randomly generated --- .../modules/clone/import/adapters/archive-reader.adapter.ts | 4 ++-- api/libs/utils/src/zip-file-extractor.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/apps/api/src/modules/clone/import/adapters/archive-reader.adapter.ts b/api/apps/api/src/modules/clone/import/adapters/archive-reader.adapter.ts index 778cede9d0..c4506e14e6 100644 --- a/api/apps/api/src/modules/clone/import/adapters/archive-reader.adapter.ts +++ b/api/apps/api/src/modules/clone/import/adapters/archive-reader.adapter.ts @@ -32,12 +32,12 @@ export class ArchiveReaderAdapter implements ArchiveReader { const exportConfigOrError = await extractFile( readableOrError.right, - new RegExp(ClonePieceRelativePaths[ClonePiece.ExportConfig].config), + ClonePieceRelativePaths[ClonePiece.ExportConfig].config, ); if (isLeft(exportConfigOrError)) return left(archiveCorrupted); const exportConfig = JSON.parse(exportConfigOrError.right); - const resourceId = new ResourceId(exportConfig.resourceId); + const resourceId = ResourceId.create(); const resourceKind = exportConfig.resourceKind; const validResourceKind = Object.values(ResourceKind).includes( diff --git a/api/libs/utils/src/zip-file-extractor.ts b/api/libs/utils/src/zip-file-extractor.ts index bd7be8cb6f..c1392e7aad 100644 --- a/api/libs/utils/src/zip-file-extractor.ts +++ b/api/libs/utils/src/zip-file-extractor.ts @@ -6,11 +6,11 @@ export const extractFileFailed = Symbol('Extract file failed'); export async function extractFile( readable: Readable, - fileName: RegExp, + fileRelativePath: string, ): Promise> { return new Promise>((resolve) => { readable - .pipe(unzipper.ParseOne(fileName)) + .pipe(unzipper.ParseOne(new RegExp(fileRelativePath))) .on('entry', async (entry: unzipper.Entry) => { const buffer = await entry.buffer(); resolve(right(buffer.toString())); From fe66855cb97a7efbfd9ef54820ea9e7b8af38c57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daute=20Rodr=C3=ADguez=20Rodr=C3=ADguez?= Date: Wed, 16 Feb 2022 13:35:01 +0000 Subject: [PATCH 2/3] feat: project metadata piece importer --- .../pieces-importers.module.ts | 7 +- .../project-metadata.piece-importer.ts | 82 ++++++++++++++++--- 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/api/apps/geoprocessing/src/import/pieces-importers/pieces-importers.module.ts b/api/apps/geoprocessing/src/import/pieces-importers/pieces-importers.module.ts index d1ebd65717..ac39cbd600 100644 --- a/api/apps/geoprocessing/src/import/pieces-importers/pieces-importers.module.ts +++ b/api/apps/geoprocessing/src/import/pieces-importers/pieces-importers.module.ts @@ -1,4 +1,4 @@ -import { Module } from '@nestjs/common'; +import { Logger, Module, Scope } from '@nestjs/common'; import { FileRepositoryModule } from '@marxan/files-repository'; import { TypeOrmModule } from '@nestjs/typeorm'; import { geoprocessingConnections } from '@marxan-geoprocessing/ormconfig'; @@ -9,6 +9,9 @@ import { ProjectMetadataPieceImporter } from './project-metadata.piece-importer' FileRepositoryModule, TypeOrmModule.forFeature([], geoprocessingConnections.apiDB), ], - providers: [ProjectMetadataPieceImporter], + providers: [ + ProjectMetadataPieceImporter, + { provide: Logger, useClass: Logger, scope: Scope.TRANSIENT }, + ], }) export class PiecesImportersModule {} diff --git a/api/apps/geoprocessing/src/import/pieces-importers/project-metadata.piece-importer.ts b/api/apps/geoprocessing/src/import/pieces-importers/project-metadata.piece-importer.ts index b4091c3083..84ce240163 100644 --- a/api/apps/geoprocessing/src/import/pieces-importers/project-metadata.piece-importer.ts +++ b/api/apps/geoprocessing/src/import/pieces-importers/project-metadata.piece-importer.ts @@ -1,15 +1,15 @@ -import { Injectable } from '@nestjs/common'; -import { InjectEntityManager } from '@nestjs/typeorm'; -import { EntityManager } from 'typeorm'; - +import { geoprocessingConnections } from '@marxan-geoprocessing/ormconfig'; import { ClonePiece, ImportJobInput, ImportJobOutput } from '@marxan/cloning'; +import { ProjectMetadataContent } from '@marxan/cloning/infrastructure/clone-piece-data/project-metadata'; import { FileRepository } from '@marxan/files-repository'; - -import { geoprocessingConnections } from '@marxan-geoprocessing/ormconfig'; - +import { extractFile } from '@marxan/utils'; +import { Injectable, Logger } from '@nestjs/common'; +import { InjectEntityManager } from '@nestjs/typeorm'; +import { isLeft } from 'fp-ts/lib/Either'; +import { EntityManager } from 'typeorm'; import { - PieceImportProvider, ImportPieceProcessor, + PieceImportProvider, } from '../pieces/import-piece-processor'; @Injectable() @@ -19,13 +19,75 @@ export class ProjectMetadataPieceImporter implements ImportPieceProcessor { private readonly fileRepository: FileRepository, @InjectEntityManager(geoprocessingConnections.apiDB) private readonly entityManager: EntityManager, - ) {} + private readonly logger: Logger, + ) { + this.logger.setContext(ProjectMetadataPieceImporter.name); + } isSupported(piece: ClonePiece): boolean { return piece === ClonePiece.ProjectMetadata; } + private async getRandomOrganizationId(): Promise { + const [{ id }]: [{ id: string }] = await this.entityManager.query(` + SELECT id FROM organizations LIMIT 1 + `); + return id; + } + async run(input: ImportJobInput): Promise { - throw new Error('Missing implementation'); + const { uris, resourceId } = input; + + if (uris.length !== 1) { + const errorMessage = `uris array has an unexpected amount of elements: ${uris.length}`; + this.logger.error(errorMessage); + throw new Error(errorMessage); + } + const [projectMetadataLocation] = uris; + + const readableOrError = await this.fileRepository.get( + projectMetadataLocation.uri, + ); + if (isLeft(readableOrError)) { + const errorMessage = `Zip file not found: ${projectMetadataLocation.uri}`; + this.logger.error(errorMessage); + throw new Error(errorMessage); + } + + const stringProjectMetadataOrError = await extractFile( + readableOrError.right, + projectMetadataLocation.relativePath, + ); + if (isLeft(stringProjectMetadataOrError)) { + const errorMessage = `Project metadata file extraction failed: ${projectMetadataLocation.relativePath}`; + this.logger.error(errorMessage); + throw new Error(errorMessage); + } + + // TODO OrganizationId should be specified when creating the Import + const organizationId = await this.getRandomOrganizationId(); + const projectMetadata: ProjectMetadataContent = JSON.parse( + stringProjectMetadataOrError.right, + ); + + await this.entityManager.query( + ` + INSERT INTO projects(id, name, description, organization_id) + VALUES ($1, $2, $3, $4) + `, + [ + resourceId, + projectMetadata.name, + projectMetadata.description, + organizationId, + ], + ); + + return { + importId: input.importId, + componentId: input.componentId, + resourceId: input.resourceId, + piece: input.piece, + }; } } From ca7f17dbd638b0bef8b03ee6094d7489f23e8582 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daute=20Rodr=C3=ADguez=20Rodr=C3=ADguez?= Date: Thu, 17 Feb 2022 08:34:23 +0000 Subject: [PATCH 3/3] ref: PR review tweaks --- .../project-metadata.piece-importer.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/api/apps/geoprocessing/src/import/pieces-importers/project-metadata.piece-importer.ts b/api/apps/geoprocessing/src/import/pieces-importers/project-metadata.piece-importer.ts index 84ce240163..f8b76adabf 100644 --- a/api/apps/geoprocessing/src/import/pieces-importers/project-metadata.piece-importer.ts +++ b/api/apps/geoprocessing/src/import/pieces-importers/project-metadata.piece-importer.ts @@ -36,7 +36,7 @@ export class ProjectMetadataPieceImporter implements ImportPieceProcessor { } async run(input: ImportJobInput): Promise { - const { uris, resourceId } = input; + const { uris, resourceId, piece } = input; if (uris.length !== 1) { const errorMessage = `uris array has an unexpected amount of elements: ${uris.length}`; @@ -49,7 +49,7 @@ export class ProjectMetadataPieceImporter implements ImportPieceProcessor { projectMetadataLocation.uri, ); if (isLeft(readableOrError)) { - const errorMessage = `Zip file not found: ${projectMetadataLocation.uri}`; + const errorMessage = `File with piece data for ${piece}/${resourceId} is not available at ${projectMetadataLocation.uri}`; this.logger.error(errorMessage); throw new Error(errorMessage); } @@ -64,7 +64,11 @@ export class ProjectMetadataPieceImporter implements ImportPieceProcessor { throw new Error(errorMessage); } - // TODO OrganizationId should be specified when creating the Import + // TODO As we don't handle organizations for the time being, + // the imported/cloned project is homed arbitrarily within an + // existing organization. Once proper handling of organizations + // is added, users may be able to specify within which organization + // an imported/cloned project should be created. const organizationId = await this.getRandomOrganizationId(); const projectMetadata: ProjectMetadataContent = JSON.parse( stringProjectMetadataOrError.right,