From 34235763a28462e784224001a769b21ea88896ad Mon Sep 17 00:00:00 2001 From: Yulia Belyakova Date: Mon, 31 Jul 2023 20:00:48 +0200 Subject: [PATCH 1/2] Add logs and cte optimization to csv features upload --- .../import/features-amounts-upload.service.ts | 45 ++++++++++++++++--- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/api/apps/api/src/modules/geo-features/import/features-amounts-upload.service.ts b/api/apps/api/src/modules/geo-features/import/features-amounts-upload.service.ts index e99665c53d..82852f273c 100644 --- a/api/apps/api/src/modules/geo-features/import/features-amounts-upload.service.ts +++ b/api/apps/api/src/modules/geo-features/import/features-amounts-upload.service.ts @@ -51,6 +51,7 @@ export class FeatureAmountUploadService { let newFeaturesFromCsvUpload; try { + this.logger.log(`Parsing csv file and saving data to temporary storage`); await this.events.createEvent(data); // saving feature data to temporary table const featuresRegistry = await this.saveCsvToRegistry( @@ -63,6 +64,8 @@ export class FeatureAmountUploadService { } // Saving new features to apiDB 'features' table + this.logger.log(`Parsing csv file`); + newFeaturesFromCsvUpload = await this.saveNewFeaturesFromCsvUpload( apiQueryRunner, featuresRegistry.right.id, @@ -70,6 +73,9 @@ export class FeatureAmountUploadService { ); // Saving new features amounts and geoms to geoDB 'features_amounts' table + this.logger.log( + `Saving uploaded features data from temporary table to permanent tables`, + ); await this.saveNewFeaturesAmountsFromCsvUpload( newFeaturesFromCsvUpload, apiQueryRunner, @@ -79,12 +85,15 @@ export class FeatureAmountUploadService { ); // Removing temporary data from apiDB uploads tables + this.logger.log( + `Removing data from temporary tables after successful upload`, + ); await apiQueryRunner.manager.delete(FeatureAmountUploadRegistry, { id: featuresRegistry.right.id, }); // Setting project source to legacy-import to create puvspr.dat files from pre-calculated amounts, to allow to use new features after upload - + this.logger.log(`Updating project sources value`); await this.updateProjectSources( data.projectId, ProjectSourcesEnum.legacyImport, @@ -122,9 +131,14 @@ export class FeatureAmountUploadService { queryRunner: QueryRunner, ): Promise { try { + this.logger.log(`Parsing csv file`); + const parsedFile = await featureAmountCsvParser(data.fileBuffer); const { featureNames, puids } = this.getFeatureNamesAndPuids(parsedFile); + + this.logger.log(`Validating parsed csv file`); + if ( await this.areFeatureNamesNotAlreadyUsedInProject( data.projectId, @@ -138,6 +152,7 @@ export class FeatureAmountUploadService { return left(unknownPuidsInFeatureAmountCsvUpload); } + this.logger.log(`Saving parsed data to temporary storage`); const importedRegistry = await this.saveFeaturesToRegistry( parsedFile, data.projectId, @@ -163,13 +178,16 @@ export class FeatureAmountUploadService { features, CHUNK_SIZE_FOR_BATCH_APIDB_OPERATIONS, ); + + this.logger.log(`Saving a new upload data to temporary table`); const newUpload = await entityManager .getRepository(FeatureAmountUploadRegistry) .save({ projectId, userId, }); - for (const chunk of featuresChunks) { + for (const [index, chunk] of featuresChunks.entries()) { + this.logger.log(`Inserting chunk ${index} to temporary table`); await entityManager .createQueryBuilder() .insert() @@ -177,6 +195,7 @@ export class FeatureAmountUploadService { .values(chunk.map((feature) => ({ ...feature, upload: newUpload }))) .execute(); } + this.logger.log(`New upload data saved to temporary table`); return newUpload; } @@ -219,7 +238,8 @@ export class FeatureAmountUploadService { uploadId: string, projectId: string, ) { - for (const newFeature of newFeaturesFromCsvUpload) { + for (const [index, newFeature] of newFeaturesFromCsvUpload.entries()) { + this.logger.log(`Getting feature amounts for feature ${index}`); const featureAmounts = await apiQueryRunner.manager .createQueryBuilder() .select(['fa.puid', 'fa.amount']) @@ -235,9 +255,15 @@ export class FeatureAmountUploadService { CHUNK_SIZE_FOR_BATCH_APIDB_OPERATIONS, ); - for (const featureChunk of featuresChunks) { + for (const [amountIndex, featureChunk] of featuresChunks.entries()) { + this.logger.log( + `Saving chunk ${amountIndex} of amounts of feature ${index}`, + ); const firstParameterNumber = 2; const parameters: any[] = [projectId]; + this.logger.log( + `Generating values to insert for chunk ${amountIndex} of amounts of feature ${index}`, + ); const valuesToInsert = featureChunk.map((featureAmount, index) => { parameters.push( ...[ @@ -260,9 +286,13 @@ export class FeatureAmountUploadService { ) `; }); + + this.logger.log( + `Inserting amount values of chunk ${amountIndex} of amounts of feature ${index} into geoDB`, + ); await geoQueryRunner.manager.query( ` - WITH project_pus AS ( + WITH project_pus AS NOT MATERIALIZED ( SELECT ppu.id, ppu.puid, pug.the_geom FROM projects_pu ppu JOIN planning_units_geom pug ON pug.id = ppu.geom_id WHERE ppu.project_id = $1 ) INSERT INTO features_data (the_geom, feature_id, amount, project_pu_id) @@ -272,8 +302,13 @@ export class FeatureAmountUploadService { `, parameters, ); + this.logger.log( + `Chunk ${amountIndex} of amounts of feature ${index} saved to geoDB features_data`, + ); } + this.logger.log(`All chunks of feature ${index} saved`); } + this.logger.log(`All features data saved`); } private async areFeatureNamesNotAlreadyUsedInProject( From bc6ac63c89bd7c346945f880377f91d79aa6c560 Mon Sep 17 00:00:00 2001 From: Yulia Belyakova Date: Tue, 1 Aug 2023 17:07:12 +0200 Subject: [PATCH 2/2] Update logs for csv features upload --- .../import/features-amounts-upload.service.ts | 54 ++++++++++++------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/api/apps/api/src/modules/geo-features/import/features-amounts-upload.service.ts b/api/apps/api/src/modules/geo-features/import/features-amounts-upload.service.ts index 82852f273c..dc58a9a09a 100644 --- a/api/apps/api/src/modules/geo-features/import/features-amounts-upload.service.ts +++ b/api/apps/api/src/modules/geo-features/import/features-amounts-upload.service.ts @@ -51,7 +51,9 @@ export class FeatureAmountUploadService { let newFeaturesFromCsvUpload; try { - this.logger.log(`Parsing csv file and saving data to temporary storage`); + this.logger.log( + `Starting process of parsing csv file and saving amount data to temporary storage`, + ); await this.events.createEvent(data); // saving feature data to temporary table const featuresRegistry = await this.saveCsvToRegistry( @@ -64,17 +66,16 @@ export class FeatureAmountUploadService { } // Saving new features to apiDB 'features' table - this.logger.log(`Parsing csv file`); - + this.logger.log(`Saving new features to (apiBD).features table...`); newFeaturesFromCsvUpload = await this.saveNewFeaturesFromCsvUpload( apiQueryRunner, featuresRegistry.right.id, data.projectId, ); - + this.logger.log(`New features saved in (apiBD).features table`); // Saving new features amounts and geoms to geoDB 'features_amounts' table this.logger.log( - `Saving uploaded features data from temporary table to permanent tables`, + `Starting the process of saving new features amounts and geoms to (geoDB).features_amounts table`, ); await this.saveNewFeaturesAmountsFromCsvUpload( newFeaturesFromCsvUpload, @@ -86,20 +87,24 @@ export class FeatureAmountUploadService { // Removing temporary data from apiDB uploads tables this.logger.log( - `Removing data from temporary tables after successful upload`, + `Removing data from temporary tables after successful upload...`, ); await apiQueryRunner.manager.delete(FeatureAmountUploadRegistry, { id: featuresRegistry.right.id, }); + this.logger.log( + `Upload temporary data removed from apiDB uploads tables`, + ); // Setting project source to legacy-import to create puvspr.dat files from pre-calculated amounts, to allow to use new features after upload - this.logger.log(`Updating project sources value`); + this.logger.log(`Updating project sources value to legacy-import...`); await this.updateProjectSources( data.projectId, ProjectSourcesEnum.legacyImport, apiQueryRunner.manager, ); + this.logger.log(`Csv file upload process finished successfully`); // Committing transaction await apiQueryRunner.commitTransaction(); @@ -131,13 +136,13 @@ export class FeatureAmountUploadService { queryRunner: QueryRunner, ): Promise { try { - this.logger.log(`Parsing csv file`); + this.logger.log(`Parsing csv file...`); const parsedFile = await featureAmountCsvParser(data.fileBuffer); const { featureNames, puids } = this.getFeatureNamesAndPuids(parsedFile); - this.logger.log(`Validating parsed csv file`); + this.logger.log(`Validating parsed csv file...`); if ( await this.areFeatureNamesNotAlreadyUsedInProject( @@ -152,7 +157,7 @@ export class FeatureAmountUploadService { return left(unknownPuidsInFeatureAmountCsvUpload); } - this.logger.log(`Saving parsed data to temporary storage`); + this.logger.log(`Saving parsed data to temporary storage...`); const importedRegistry = await this.saveFeaturesToRegistry( parsedFile, data.projectId, @@ -179,7 +184,7 @@ export class FeatureAmountUploadService { CHUNK_SIZE_FOR_BATCH_APIDB_OPERATIONS, ); - this.logger.log(`Saving a new upload data to temporary table`); + this.logger.log(`Saving a new upload data to temporary table...`); const newUpload = await entityManager .getRepository(FeatureAmountUploadRegistry) .save({ @@ -187,7 +192,7 @@ export class FeatureAmountUploadService { userId, }); for (const [index, chunk] of featuresChunks.entries()) { - this.logger.log(`Inserting chunk ${index} to temporary table`); + this.logger.log(`Inserting chunk ${index} to temporary table...`); await entityManager .createQueryBuilder() .insert() @@ -195,7 +200,7 @@ export class FeatureAmountUploadService { .values(chunk.map((feature) => ({ ...feature, upload: newUpload }))) .execute(); } - this.logger.log(`New upload data saved to temporary table`); + this.logger.log(`New csv upload data from saved to temporary tables`); return newUpload; } @@ -239,7 +244,11 @@ export class FeatureAmountUploadService { projectId: string, ) { for (const [index, newFeature] of newFeaturesFromCsvUpload.entries()) { - this.logger.log(`Getting feature amounts for feature ${index}`); + this.logger.log( + `Getting feature amounts for feature number ${index + 1}: ${ + newFeature.feature_class_name + }`, + ); const featureAmounts = await apiQueryRunner.manager .createQueryBuilder() .select(['fa.puid', 'fa.amount']) @@ -255,14 +264,17 @@ export class FeatureAmountUploadService { CHUNK_SIZE_FOR_BATCH_APIDB_OPERATIONS, ); + this.logger.log( + `Feature data divided into ${featuresChunks.length} chunks`, + ); for (const [amountIndex, featureChunk] of featuresChunks.entries()) { this.logger.log( - `Saving chunk ${amountIndex} of amounts of feature ${index}`, + `Starting the process of saving chunk with index ${amountIndex} of amounts of feature ${newFeature.feature_class_name}...`, ); const firstParameterNumber = 2; const parameters: any[] = [projectId]; this.logger.log( - `Generating values to insert for chunk ${amountIndex} of amounts of feature ${index}`, + `Generating values to insert for chunk with index ${amountIndex}...`, ); const valuesToInsert = featureChunk.map((featureAmount, index) => { parameters.push( @@ -288,7 +300,7 @@ export class FeatureAmountUploadService { }); this.logger.log( - `Inserting amount values of chunk ${amountIndex} of amounts of feature ${index} into geoDB`, + `Inserting amount values of chunk with index ${amountIndex} into (geoDB).features_data table...`, ); await geoQueryRunner.manager.query( ` @@ -303,12 +315,14 @@ export class FeatureAmountUploadService { parameters, ); this.logger.log( - `Chunk ${amountIndex} of amounts of feature ${index} saved to geoDB features_data`, + `Chunk with index ${amountIndex} saved to (geoDB).features_data`, ); } - this.logger.log(`All chunks of feature ${index} saved`); + this.logger.log( + `All chunks of feature ${newFeature.feature_class_name} saved`, + ); } - this.logger.log(`All features data saved`); + this.logger.log(`All new features data saved to (geoDB).features_data`); } private async areFeatureNamesNotAlreadyUsedInProject(