diff --git a/api/apps/geoprocessing/src/migrations/geoprocessing/1657795596079-CreateFeaturesDataCleanupPreparation.ts b/api/apps/geoprocessing/src/migrations/geoprocessing/1657795596079-CreateFeaturesDataCleanupPreparation.ts new file mode 100644 index 0000000000..56d1a91bf8 --- /dev/null +++ b/api/apps/geoprocessing/src/migrations/geoprocessing/1657795596079-CreateFeaturesDataCleanupPreparation.ts @@ -0,0 +1,17 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class CreateFeaturesDataCleanupPreparation1657795596079 + implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + CREATE TABLE features_data_cleanup_preparation ( + feature_id uuid PRIMARY KEY + );`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + DROP TABLE features_data_cleanup_preparation; + `); + } +} diff --git a/api/apps/geoprocessing/src/modules/cleanup-tasks/cleanup-tasks.service.ts b/api/apps/geoprocessing/src/modules/cleanup-tasks/cleanup-tasks.service.ts index a876d57e20..f72fc7bc03 100644 --- a/api/apps/geoprocessing/src/modules/cleanup-tasks/cleanup-tasks.service.ts +++ b/api/apps/geoprocessing/src/modules/cleanup-tasks/cleanup-tasks.service.ts @@ -17,7 +17,7 @@ export class CleanupTasksService implements CleanupTasks { constructor( @InjectEntityManager(geoprocessingConnections.apiDB) private readonly apiEntityManager: EntityManager, - @InjectEntityManager(geoprocessingConnections.apiDB) + @InjectEntityManager(geoprocessingConnections.default) private readonly geoEntityManager: EntityManager, ) {} @@ -36,7 +36,7 @@ export class CleanupTasksService implements CleanupTasks { // Start cleaning up process inside transaction await this.geoEntityManager.transaction(async (entityManager) => { // Truncate table to be sure that not any projectId is inside before operation - await this.geoEntityManager.query( + await entityManager.query( `TRUNCATE TABLE project_geodata_cleanup_preparation;`, ); @@ -55,21 +55,21 @@ export class CleanupTasksService implements CleanupTasks { // For every related entity, we look for non-matching ids inside entity table // and compare it with intermediate projectId table to delete records that are not there - await this.geoEntityManager.query( + await entityManager.query( `DELETE FROM planning_areas pa WHERE pa.project_id IS NOT NULL AND pa.project_id NOT IN ( SELECT pgcp.project_id FROM project_geodata_cleanup_preparation pgcp );`, ); - await this.geoEntityManager.query( + await entityManager.query( `DELETE FROM wdpa WHERE wdpa.project_id IS NOT NULL AND wdpa.project_id NOT IN ( SELECT pgcp.project_id FROM project_geodata_cleanup_preparation pgcp );`, ); - await this.geoEntityManager.query( + await entityManager.query( `DELETE FROM projects_pu ppu WHERE ppu.project_id IS NOT NULL AND ppu.project_id NOT IN ( @@ -84,7 +84,7 @@ export class CleanupTasksService implements CleanupTasks { );`, ); - await this.geoEntityManager.query( + await entityManager.query( `TRUNCATE TABLE project_geodata_cleanup_preparation;`, ); }); @@ -102,7 +102,7 @@ export class CleanupTasksService implements CleanupTasks { }); await this.geoEntityManager.transaction(async (entityManager) => { - await this.geoEntityManager.query( + await entityManager.query( `TRUNCATE TABLE scenario_geodata_cleanup_preparation;`, ); @@ -118,35 +118,35 @@ export class CleanupTasksService implements CleanupTasks { ); } - await this.geoEntityManager.query( + await entityManager.query( `DELETE FROM scenario_features_data sfd WHERE sfd.scenario_id IS NOT NULL AND sfd.scenario_id NOT IN ( SELECT sgcp.scenario_id FROM scenario_geodata_cleanup_preparation sgcp );`, ); - await this.geoEntityManager.query( + await entityManager.query( `DELETE FROM blm_final_results bfr WHERE bfr.scenario_id IS NOT NULL AND bfr.scenario_id NOT IN ( SELECT sgcp.scenario_id FROM scenario_geodata_cleanup_preparation sgcp );`, ); - await this.geoEntityManager.query( + await entityManager.query( `DELETE FROM blm_partial_results bpr WHERE bpr.scenario_id IS NOT NULL AND bpr.scenario_id NOT IN ( SELECT sgcp.scenario_id FROM scenario_geodata_cleanup_preparation sgcp );`, ); - await this.geoEntityManager.query( + await entityManager.query( `DELETE FROM marxan_execution_metadata mem WHERE mem.scenarioId IS NOT NULL AND mem.scenarioId NOT IN ( SELECT sgcp.scenario_id FROM scenario_geodata_cleanup_preparation sgcp );`, ); - await this.geoEntityManager.query( + await entityManager.query( `DELETE FROM scenarios_pu_data spd WHERE spd.scenario_id IS NOT NULL AND spd.scenario_id NOT IN ( @@ -154,12 +154,53 @@ export class CleanupTasksService implements CleanupTasks { );`, ); - await this.geoEntityManager.query( + await entityManager.query( `TRUNCATE TABLE scenario_geodata_cleanup_preparation;`, ); }); } + async cleanupFeaturesDanglingData() { + const featureIds = await this.apiEntityManager + .createQueryBuilder() + .from('features', 'f') + .select(['id']) + .getRawMany() + .then((result) => result.map((i) => i.id)) + .catch((error) => { + throw new Error(error); + }); + + await this.geoEntityManager.transaction(async (entityManager) => { + await entityManager.query( + `TRUNCATE TABLE features_data_cleanup_preparation;`, + ); + + for (const [, summaryChunks] of chunk( + featureIds, + CHUNK_SIZE_FOR_BATCH_DB_OPERATIONS, + ).entries()) { + await entityManager.insert( + 'features_data_cleanup_preparation', + summaryChunks.map((chunk: string) => ({ + id: chunk, + })), + ); + } + + await entityManager.query( + `DELETE FROM features_data fa + WHERE fa.feature_id NOT IN ( + SELECT fdcp.id FROM features_data_cleanup_preparation fdcp + );`, + ); + + await entityManager.query( + `TRUNCATE TABLE features_data_cleanup_preparation;`, + ); + }); + } + @Cron(cronJobInterval) async handleCron() { this.logger.debug( @@ -168,5 +209,6 @@ export class CleanupTasksService implements CleanupTasks { await this.cleanupProjectsDanglingData(); await this.cleanupScenariosDanglingData(); + await this.cleanupFeaturesDanglingData(); } }