feat(custom-pu-grid): custom pu grid geometry

api/apps/api/src/modules/scenarios/planning-units/scenario-planning-units-linker-service.ts

@@ -81,9 +81,6 @@ export class ScenarioPlanningUnitsLinkerService {
     project: Project,
   ): QueryPartsForLinker | undefined {
     /**
-     * @TODO selection by planning_units_geom.project_id needs project_id column
-     * to be added and set.
-     *
      * We still intersect in case planning unit grid is not fully included in
      * the planning area.
      */

api/apps/api/test/jest-e2e.json

@@ -66,6 +66,8 @@
     "@marxan/iucn/(.*)": "<rootDir>/../../../libs/iucn/src/$1",
     "@marxan/iucn": "<rootDir>/../../../libs/iucn/src",
     "@marxan/shapefile-converter/(.*)": "<rootDir>/../../../libs/shapefile-converter/src/$1",
-    "@marxan/shapefile-converter": "<rootDir>/../../../libs/shapefile-converter/src"
+    "@marxan/shapefile-converter": "<rootDir>/../../../libs/shapefile-converter/src",
+    "@marxan/planning-units-grid/(.*)": "<rootDir>/../../../libs/planning-units-grid/src/$1",
+    "@marxan/planning-units-grid": "<rootDir>/../../../libs/planning-units-grid/src"
   }
 }

api/apps/geoprocessing/src/app.module.ts

@@ -16,6 +16,7 @@ import { PlanningAreaModule } from '@marxan-geoprocessing/modules/planning-area/
 import { MarxanSandboxedRunnerModule } from '@marxan-geoprocessing/marxan-sandboxed-runner/marxan-sandboxed-runner.module';
 import { ScenariosModule } from '@marxan-geoprocessing/modules/scenarios/scenarios.module';
 import { ScenarioProtectedAreaCalculationModule } from '@marxan-geoprocessing/modules/scenario-protected-area-calculation/scenario-protected-area-calculation.module';
+import { PlanningUnitsGridModule } from '@marxan-geoprocessing/modules/planning-units-grid';
 
 @Module({
   imports: [
@@ -40,6 +41,7 @@ import { ScenarioProtectedAreaCalculationModule } from '@marxan-geoprocessing/mo
     PlanningAreaModule,
     MarxanSandboxedRunnerModule,
     ScenariosModule,
+    PlanningUnitsGridModule,
   ],
   controllers: [AppController],
   providers: [AppService],

api/apps/geoprocessing/src/migrations/geoprocessing/1631080265630-PuGeometryUniqueWithinProject.ts

@@ -0,0 +1,26 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class PuGeometryUniqueWithinProject1631080265630
+  implements MigrationInterface {
+  async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+      alter table planning_units_geom
+        drop constraint planning_units_geom_the_geom_type_key;
+
+      alter table planning_units_geom
+        add constraint planning_units_geom_the_geom_type_key
+          unique (the_geom, type, project_id);
+    `);
+  }
+
+  async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+      alter table planning_units_geom
+        drop constraint planning_units_geom_the_geom_type_key;
+
+      alter table planning_units_geom
+        add constraint planning_units_geom_the_geom_type_key
+          unique (the_geom, type);
+    `);
+  }
+}

api/apps/geoprocessing/src/modules/planning-units-grid/grid-geojson-validator.spec.ts

@@ -0,0 +1,81 @@
+import { Test } from '@nestjs/testing';
+import { FeatureCollection, Geometry } from 'geojson';
+import {
+  GridGeoJsonValidator,
+  invalidFeatureGeometry,
+  notFeatureCollections,
+} from './grid-geojson-validator';
+
+let sut: GridGeoJsonValidator;
+
+beforeEach(async () => {
+  const sandbox = await Test.createTestingModule({
+    providers: [GridGeoJsonValidator],
+  }).compile();
+  sut = sandbox.get(GridGeoJsonValidator);
+});
+
+test(`providing non feature collection`, () => {
+  expect(sut.validate(nonFeatureCollection())).toEqual({
+    _tag: 'Left',
+    left: notFeatureCollections,
+  });
+});
+
+test(`providing non polygons within collection`, () => {
+  expect(sut.validate(featureCollectionWithLine())).toEqual({
+    _tag: 'Left',
+    left: invalidFeatureGeometry,
+  });
+});
+
+test(`providing feature collection with polygons only`, () => {
+  expect(sut.validate(featureCollectionWithPolygons())).toEqual({
+    _tag: 'Right',
+    right: featureCollectionWithPolygons(),
+  });
+});
+
+const nonFeatureCollection = (): Geometry => ({
+  type: 'Point',
+  bbox: [0, 0, 0, 0, 0, 0],
+  coordinates: [],
+});
+
+const featureCollectionWithLine = (): FeatureCollection => ({
+  type: 'FeatureCollection',
+  bbox: [0, 0, 0, 0, 0, 0],
+  features: [
+    {
+      type: 'Feature',
+      properties: {},
+      geometry: {
+        type: 'Polygon',
+        coordinates: [],
+      },
+    },
+    {
+      type: 'Feature',
+      properties: {},
+      geometry: {
+        type: 'MultiPolygon',
+        coordinates: [],
+      },
+    },
+  ],
+});
+
+const featureCollectionWithPolygons = (): FeatureCollection => ({
+  type: 'FeatureCollection',
+  bbox: [0, 0, 0, 0, 0, 0],
+  features: [
+    {
+      type: 'Feature',
+      properties: {},
+      geometry: {
+        type: 'Polygon',
+        coordinates: [],
+      },
+    },
+  ],
+});

api/apps/geoprocessing/src/modules/planning-units-grid/grid-geojson-validator.ts

@@ -0,0 +1,45 @@
+import { Injectable } from '@nestjs/common';
+import {
+  FeatureCollection,
+  GeoJSON,
+  GeoJsonProperties,
+  Geometry,
+  Polygon,
+} from 'geojson';
+import { Either, left, right } from 'fp-ts/Either';
+
+export const notFeatureCollections = Symbol(`not a feature collections`);
+export const invalidFeatureGeometry = Symbol(`can only contain polygons`);
+
+export type ValidationError =
+  | typeof notFeatureCollections
+  | typeof invalidFeatureGeometry;
+
+@Injectable()
+export class GridGeoJsonValidator {
+  validate(
+    geo: GeoJSON,
+  ): Either<ValidationError, FeatureCollection<Polygon, GeoJsonProperties>> {
+    if (!this.isFeatureCollection(geo)) {
+      return left(notFeatureCollections);
+    }
+
+    if (!this.hasOnlyPolygons(geo)) {
+      return left(invalidFeatureGeometry);
+    }
+
+    return right(geo);
+  }
+
+  private isFeatureCollection(geo: GeoJSON): geo is FeatureCollection {
+    return geo.type === 'FeatureCollection';
+  }
+
+  private hasOnlyPolygons(
+    geoJson: FeatureCollection<Geometry, GeoJsonProperties>,
+  ): geoJson is FeatureCollection<Polygon, GeoJsonProperties> {
+    return geoJson.features.every(
+      (feature) => feature.geometry.type === 'Polygon',
+    );
+  }
+}

api/apps/geoprocessing/src/modules/planning-units-grid/index.ts

@@ -0,0 +1 @@
+export { PlanningUnitsGridModule } from './planning-units-grid.module';

api/apps/geoprocessing/src/modules/planning-units-grid/planning-units-grid.module.ts

@@ -0,0 +1,13 @@
+import { Module } from '@nestjs/common';
+import { WorkerModule } from '@marxan-geoprocessing/modules/worker';
+import { ShapefilesModule } from '@marxan/shapefile-converter';
+import { TypeOrmModule } from '@nestjs/typeorm';
+
+import { PlanningUnitsGridProcessor } from './planning-units-grid.processor';
+import { GridGeoJsonValidator } from './grid-geojson-validator';
+
+@Module({
+  imports: [WorkerModule, ShapefilesModule, TypeOrmModule.forFeature([])],
+  providers: [PlanningUnitsGridProcessor, GridGeoJsonValidator],
+})
+export class PlanningUnitsGridModule {}

api/apps/geoprocessing/src/modules/planning-units-grid/planning-units-grid.processor.ts

@@ -0,0 +1,76 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { Job, Worker } from 'bullmq';
+import { plainToClass } from 'class-transformer';
+import { validateSync } from 'class-validator';
+import { InjectEntityManager } from '@nestjs/typeorm';
+import { EntityManager } from 'typeorm';
+
+import { JobInput, JobOutput, queueName } from '@marxan/planning-units-grid';
+import { ShapefileService } from '@marxan/shapefile-converter';
+import {
+  WorkerBuilder,
+  WorkerProcessor,
+} from '@marxan-geoprocessing/modules/worker';
+
+import { ShapeType } from '@marxan-jobs/planning-unit-geometry';
+import { GridGeoJsonValidator } from './grid-geojson-validator';
+import { isLeft } from 'fp-ts/Either';
+
+@Injectable()
+export class PlanningUnitsGridProcessor
+  implements WorkerProcessor<JobInput, JobOutput> {
+  private readonly worker: Worker;
+  private readonly logger = new Logger(this.constructor.name);
+
+  constructor(
+    @InjectEntityManager() private readonly entityManager: EntityManager,
+    private readonly shapefileConverter: ShapefileService,
+    private readonly gridGeoJsonValidator: GridGeoJsonValidator,
+    workerBuilder: WorkerBuilder,
+  ) {
+    this.worker = workerBuilder.build(queueName, this);
+  }
+
+  async process({
+    data: { projectId, shapefile },
+  }: Job<JobInput, JobOutput>): Promise<JobOutput> {
+    const { data: geoJson } = await this.shapefileConverter.transformToGeoJson(
+      shapefile,
+    );
+
+    const result = this.gridGeoJsonValidator.validate(geoJson);
+    if (isLeft(result)) {
+      throw new Error(result.left.toString());
+    }
+    const puGeometriesIds: { id: string }[] = await this.entityManager.query(
+      `
+        INSERT INTO "planning_units_geom"("the_geom", "type", "project_id")
+        SELECT ST_SetSRID(
+                 ST_GeomFromGeoJSON(features ->> 'geometry'),
+                 4326)::geometry,
+               $2,
+               $3
+        FROM (
+               SELECT json_array_elements($1::json -> 'features') AS features
+             ) AS f
+        ON CONFLICT ON CONSTRAINT planning_units_geom_the_geom_type_key DO UPDATE SET type = 'irregular'
+        RETURNING "id"
+      `,
+      [result.right, ShapeType.Irregular, projectId],
+    );
+    const output = plainToClass<JobOutput, JobOutput>(JobOutput, {
+      geometryIds: puGeometriesIds.map((row) => row.id),
+      projectId,
+    });
+
+    const errors = validateSync(output);
+
+    if (errors.length > 0) {
+      const errorMessage = errors.map((e) => e.toString()).join('. ');
+      this.logger.warn(`Invalid job output: ${errorMessage}`);
+      throw new Error(errorMessage);
+    }
+
+    return output;
+  }
+}

api/apps/geoprocessing/test/integration/planning-unit-grid/nam.geojson

@@ -0,0 +1 @@
+{"type":"FeatureCollection","features":[{"type":"Feature","properties":{},"geometry":{"type":"Polygon","coordinates":[[[14.468994140624998,-19.394067895396613],[15.1171875,-19.228176737766248],[15.831298828124998,-19.300774825858937],[15.809326171875,-18.490028573953296],[15.534667968749998,-18.57336188365185],[14.974365234375,-18.05186707354763],[14.403076171875,-18.250219977065594],[14.6337890625,-18.81271785640776],[14.348144531249998,-18.999802829053262],[14.293212890625,-19.373340713364044],[14.359130859375,-19.445874298215937],[14.468994140624998,-19.394067895396613]]]}},{"type":"Feature","properties":{},"geometry":{"type":"Polygon","coordinates":[[[15.853271484375,-18.490028573953296],[15.831298828124998,-19.300774825858937],[16.6552734375,-19.476950206488414],[16.798095703124996,-19.4665922322076],[17.11669921875,-19.197053439464852],[17.07275390625,-18.791917744234425],[17.237548828125,-18.760712758499565],[17.061767578125,-18.510865709091362],[16.5673828125,-18.35452552912664],[16.358642578125,-18.469188904417177],[15.853271484375,-18.490028573953296]]]}},{"type":"Feature","properties":{},"geometry":{"type":"Polygon","coordinates":[[[20.14892578125,-20.282808691330054],[20.98388671875,-20.293113447544098],[20.98388671875,-20.00432229599871],[19.962158203125,-19.99399846948549],[19.962158203125,-20.282808691330054],[20.14892578125,-20.282808691330054]]]}},{"type":"Feature","properties":{},"geometry":{"type":"Polygon","coordinates":[[[20.972900390625,-19.942369189542],[20.972900390625,-19.570142140282975],[20.137939453125,-19.528730138897643],[19.86328125,-19.611543503814232],[19.896240234375,-19.99399846948549],[20.972900390625,-19.942369189542]]]}},{"type":"Feature","properties":{},"geometry":{"type":"Polygon","coordinates":[[[21.005859375,-19.559790136497398],[20.972900390625,-19.093266636089698],[20.423583984375,-19.155546551403607],[20.214843749999996,-19.259294140463897],[20.225830078125,-19.508020154916768],[21.005859375,-19.559790136497398]]]}},{"type":"Feature","properties":{},"geometry":{"type":"Polygon","coordinates":[[[20.994873046875,-19.082884369340157],[20.994873046875,-18.44834670293207],[20.775146484375,-18.198043686762652],[20.698242187499996,-18.44834670293207],[20.56640625,-18.594188856740427],[20.56640625,-18.698285474146807],[20.489501953125,-19.134789188332523],[20.994873046875,-19.082884369340157]]]}},{"type":"Feature","properties":{},"geometry":{"type":"Polygon","coordinates":[[[20.994873046875,-18.40665471391906],[20.994873046875,-18.312810846425442],[21.07177734375,-18.312810846425442],[21.0498046875,-18.03097474989002],[20.775146484375,-18.083200903334312],[20.76416015625,-18.17716879354469],[20.994873046875,-18.40665471391906]]]}}]}

api/apps/geoprocessing/test/integration/planning-unit-grid/planning-units-grid.e2e-spec.ts

@@ -0,0 +1,18 @@
+import { FixtureType } from '@marxan/utils/tests/fixture-type';
+import { getFixtures } from './planning-units-grid.fixtures';
+
+let fixtures: FixtureType<typeof getFixtures>;
+
+beforeEach(async () => {
+  fixtures = await getFixtures();
+});
+
+test(`uploading shapefile as planning units`, async () => {
+  const shapefile = await fixtures.GivenShapefileWasUploaded();
+  const output = await fixtures.WhenConvertingShapefileToPlanningUnits(
+    shapefile,
+  );
+  await fixtures.ThenGeoJsonMatchesInput(output);
+});
+
+afterEach(async () => fixtures?.cleanup());

api/apps/geoprocessing/test/integration/planning-unit-grid/planning-units-grid.fixtures.ts

@@ -0,0 +1,85 @@
+import { bootstrapApplication } from '../../utils';
+import { getRepositoryToken } from '@nestjs/typeorm';
+import { copyFileSync, readFileSync } from 'fs';
+import { plainToClass } from 'class-transformer';
+import { v4 } from 'uuid';
+import { Job } from 'bullmq';
+import { Repository } from 'typeorm';
+
+import { AppConfig } from '@marxan-geoprocessing/utils/config.utils';
+import { PlanningUnitsGeom } from '@marxan-jobs/planning-unit-geometry';
+
+import { JobInput, JobOutput } from '@marxan/planning-units-grid';
+import { PlanningUnitsGridProcessor } from '@marxan-geoprocessing/modules/planning-units-grid/planning-units-grid.processor';
+
+export const getFixtures = async () => {
+  const app = await bootstrapApplication();
+  const sut: PlanningUnitsGridProcessor = app.get(PlanningUnitsGridProcessor);
+  const puGeoRepo: Repository<PlanningUnitsGeom> = app.get(
+    getRepositoryToken(PlanningUnitsGeom),
+  );
+  const projectId = v4();
+  return {
+    cleanup: async () => {
+      await app.close();
+    },
+    projectId,
+    GivenShapefileWasUploaded: (): JobInput['shapefile'] => {
+      const fileName = 'nam-shapefile';
+      const baseDir = AppConfig.get<string>(
+        'storage.sharedFileStorage.localPath',
+      ) as string;
+      const shapePath = baseDir + `/${fileName}.zip`;
+      copyFileSync(__dirname + `/${fileName}.zip`, shapePath);
+      return {
+        filename: fileName,
+        path: shapePath,
+        destination: baseDir,
+      };
+    },
+    WhenConvertingShapefileToPlanningUnits: async (
+      input: JobInput['shapefile'],
+    ): Promise<JobOutput> => {
+      return await sut.process(({
+        data: plainToClass<JobInput, JobInput>(JobInput, {
+          projectId,
+          shapefile: input,
+        }),
+      } as unknown) as Job<JobInput, JobOutput>);
+    },
+    ThenGeoJsonMatchesInput: async (output: JobOutput) => {
+      const underlyingGeoJson = JSON.parse(
+        readFileSync(__dirname + `/nam.geojson`, {
+          encoding: 'utf8',
+        }),
+      );
+
+      const geoJsonFromGeometries = (
+        await puGeoRepo.query(
+          `
+            select json_build_object(
+                     'type', 'FeatureCollection',
+                     'features', json_agg(ST_AsGeoJSON((t.*)::record, '', 15)::json)
+                       )
+                     from (
+                       select the_geom
+                       from planning_units_geom
+                       where type = 'irregular'
+                         and project_id = $1
+                     ) as t(geom)
+          `,
+          [projectId],
+        )
+      )[0].json_build_object;
+
+      expect(underlyingGeoJson.features).toEqual(
+        expect.arrayContaining(geoJsonFromGeometries.features),
+      );
+
+      expect(output.projectId).toEqual(projectId);
+      expect(output.geometryIds.length).toEqual(
+        underlyingGeoJson.features.length,
+      );
+    },
+  };
+};