MARXAN-1684-cleanup-features-data

api/apps/geoprocessing/src/migrations/geoprocessing/1657795596079-CreateFeaturesDataCleanupPreparation.ts

@@ -0,0 +1,17 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class CreateFeaturesDataCleanupPreparation1657795596079
+  implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+            CREATE TABLE features_data_cleanup_preparation (
+                feature_id uuid PRIMARY KEY
+              );`);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+            DROP TABLE features_data_cleanup_preparation;
+        `);
+  }
+}

api/apps/geoprocessing/src/modules/cleanup-tasks/cleanup-tasks.service.ts

@@ -17,7 +17,7 @@ export class CleanupTasksService implements CleanupTasks {
   constructor(
     @InjectEntityManager(geoprocessingConnections.apiDB)
     private readonly apiEntityManager: EntityManager,
-    @InjectEntityManager(geoprocessingConnections.apiDB)
+    @InjectEntityManager(geoprocessingConnections.default)
     private readonly geoEntityManager: EntityManager,
   ) {}
 
@@ -36,7 +36,7 @@ export class CleanupTasksService implements CleanupTasks {
     // Start cleaning up process inside transaction
     await this.geoEntityManager.transaction(async (entityManager) => {
       // Truncate table to be sure that not any projectId is inside before operation
-      await this.geoEntityManager.query(
+      await entityManager.query(
         `TRUNCATE TABLE project_geodata_cleanup_preparation;`,
       );
 
@@ -55,21 +55,21 @@ export class CleanupTasksService implements CleanupTasks {
 
       // For every related entity, we look for non-matching ids inside entity table
       // and compare it with intermediate projectId table to delete records that are not there
-      await this.geoEntityManager.query(
+      await entityManager.query(
         `DELETE FROM planning_areas pa
         WHERE pa.project_id IS NOT NULL
         AND pa.project_id NOT IN (
           SELECT pgcp.project_id FROM project_geodata_cleanup_preparation pgcp
         );`,
       );
-      await this.geoEntityManager.query(
+      await entityManager.query(
         `DELETE FROM wdpa
         WHERE wdpa.project_id IS NOT NULL
         AND wdpa.project_id NOT IN (
           SELECT pgcp.project_id FROM project_geodata_cleanup_preparation pgcp
         );`,
       );
-      await this.geoEntityManager.query(
+      await entityManager.query(
         `DELETE FROM projects_pu ppu
         WHERE ppu.project_id IS NOT NULL
         AND ppu.project_id NOT IN (
@@ -84,7 +84,7 @@ export class CleanupTasksService implements CleanupTasks {
         );`,
       );
 
-      await this.geoEntityManager.query(
+      await entityManager.query(
         `TRUNCATE TABLE project_geodata_cleanup_preparation;`,
       );
     });
@@ -102,7 +102,7 @@ export class CleanupTasksService implements CleanupTasks {
       });
 
     await this.geoEntityManager.transaction(async (entityManager) => {
-      await this.geoEntityManager.query(
+      await entityManager.query(
         `TRUNCATE TABLE scenario_geodata_cleanup_preparation;`,
       );
 
@@ -118,48 +118,89 @@ export class CleanupTasksService implements CleanupTasks {
         );
       }
 
-      await this.geoEntityManager.query(
+      await entityManager.query(
         `DELETE FROM scenario_features_data sfd
         WHERE sfd.scenario_id IS NOT NULL
         AND sfd.scenario_id NOT IN (
           SELECT sgcp.scenario_id FROM scenario_geodata_cleanup_preparation sgcp
         );`,
       );
-      await this.geoEntityManager.query(
+      await entityManager.query(
         `DELETE FROM blm_final_results bfr
         WHERE bfr.scenario_id IS NOT NULL
         AND bfr.scenario_id NOT IN (
           SELECT sgcp.scenario_id FROM scenario_geodata_cleanup_preparation sgcp
         );`,
       );
-      await this.geoEntityManager.query(
+      await entityManager.query(
         `DELETE FROM blm_partial_results bpr
         WHERE bpr.scenario_id IS NOT NULL
         AND bpr.scenario_id NOT IN (
           SELECT sgcp.scenario_id FROM scenario_geodata_cleanup_preparation sgcp
         );`,
       );
-      await this.geoEntityManager.query(
+      await entityManager.query(
         `DELETE FROM marxan_execution_metadata mem
         WHERE mem.scenarioId IS NOT NULL
         AND mem.scenarioId NOT IN (
           SELECT sgcp.scenario_id FROM scenario_geodata_cleanup_preparation sgcp
         );`,
       );
-      await this.geoEntityManager.query(
+      await entityManager.query(
         `DELETE FROM scenarios_pu_data spd
         WHERE spd.scenario_id IS NOT NULL
         AND spd.scenario_id NOT IN (
           SELECT sgcp.scenario_id FROM scenario_geodata_cleanup_preparation sgcp
         );`,
       );
 
-      await this.geoEntityManager.query(
+      await entityManager.query(
         `TRUNCATE TABLE scenario_geodata_cleanup_preparation;`,
       );
     });
   }
 
+  async cleanupFeaturesDanglingData() {
+    const featureIds = await this.apiEntityManager
+      .createQueryBuilder()
+      .from('features', 'f')
+      .select(['id'])
+      .getRawMany()
+      .then((result) => result.map((i) => i.id))
+      .catch((error) => {
+        throw new Error(error);
+      });
+
+    await this.geoEntityManager.transaction(async (entityManager) => {
+      await entityManager.query(
+        `TRUNCATE TABLE features_data_cleanup_preparation;`,
+      );
+
+      for (const [, summaryChunks] of chunk(
+        featureIds,
+        CHUNK_SIZE_FOR_BATCH_DB_OPERATIONS,
+      ).entries()) {
+        await entityManager.insert(
+          'features_data_cleanup_preparation',
+          summaryChunks.map((chunk: string) => ({
+            id: chunk,
+          })),
+        );
+      }
+
+      await entityManager.query(
+        `DELETE FROM features_data fa
+        WHERE fa.feature_id NOT IN (
+          SELECT fdcp.id FROM features_data_cleanup_preparation fdcp
+        );`,
+      );
+
+      await entityManager.query(
+        `TRUNCATE TABLE features_data_cleanup_preparation;`,
+      );
+    });
+  }
+
   @Cron(cronJobInterval)
   async handleCron() {
     this.logger.debug(
@@ -168,5 +209,6 @@ export class CleanupTasksService implements CleanupTasks {
 
     await this.cleanupProjectsDanglingData();
     await this.cleanupScenariosDanglingData();
+    await this.cleanupFeaturesDanglingData();
   }
 }