Skip to content

Commit

Permalink
Merge branch 'master' into water-quality-data
Browse files Browse the repository at this point in the history
  • Loading branch information
ericboucher authored Dec 17, 2024
2 parents 91b7e58 + 2958800 commit ca5e4b9
Show file tree
Hide file tree
Showing 21 changed files with 1,757 additions and 232 deletions.
5 changes: 5 additions & 0 deletions packages/api/ONBOARDING.md
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ The rest of the scripts are used to either augment the models with missing data
- Wind-wave data: Update hindcast data in `forecast_data` table
- `yarn update-wind-wave-date [-s siteId1 siteId2 ...]`
- `fill-noaa-nearest-point` requires optional dependency `netcdf4` to run. You may have trouble installing it with python versions at or above `3.11`. `Python 3.10.6` is tested to build the package successfully.
- `reef-check`: Upload sites, surveys and data from Reef Check. This script is currently run manually when Reef Check sends us new data. Supports 4 commands: `upload-sites`, `upload-surveys`, `upload-organisms` & `upload-substrates`. Upload sites and surveys first.

### Swagger API docs

Expand Down Expand Up @@ -184,6 +185,10 @@ refreshMaterializedView(repository)

**\*** TypeORM does not allow complex syntax on indices, so we edited the generated migration (`1622124846208-RefactorTimeSeries.ts`) to include the descending order on timestamps.

### Reef Check feature

Reef Check is an organization dedicated to the conservation of reefs and we integrate their surveys into our app. We have separate tables for their sites and surveys (`reef_check_site` and `reef_check_surveys`) Each Reef Check Site is linked to one Site, we are either creating a new one or linking to an existing one within 100m. We are also storing information about substrates, fish and diseases in `reef_check_organism` and `reef_check_substrates` related to each survey. Reef Check data is manually send to us in xlsx files and we upload them in database with the `reef-check.ts` script. Run `yarn reef-check --help` or check the script for more information.

### Jest

In order to test our application we have created both `functional` and `e2e` tests. After all tests have run we also calculate our coverage. We aim for green coverage on all api endpoints and and all util functions that are tested. There is no need to try and fully cover all functions in the codebase as many of those rely on third party libraries, which makes it complicated and unnecessary to test.
Expand Down
75 changes: 75 additions & 0 deletions packages/api/migration/1731505574037-AddReefCheckData.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
import { MigrationInterface, QueryRunner } from 'typeorm';

export class AddReefCheckData1731505574037 implements MigrationInterface {
name = 'AddReefCheckData1731505574037';

public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`CREATE TABLE "reef_check_site" ("id" character varying NOT NULL, "site_id" integer NOT NULL, "reef_name" character varying, "orientation" character varying, "country" character varying, "state_province_island" character varying, "city_town" character varying, "region" character varying, "distance_from_shore" double precision, "distance_from_nearest_river" double precision, "distance_to_nearest_popn" double precision, CONSTRAINT "REL_228715834910cd57948e2d2dbd" UNIQUE ("site_id"), CONSTRAINT "PK_aeaf0fabffa46cfae6ae194864a" PRIMARY KEY ("id"))`,
);
await queryRunner.query(
`CREATE INDEX "IDX_228715834910cd57948e2d2dbd" ON "reef_check_site" ("site_id") `,
);
await queryRunner.query(
`CREATE TABLE "reef_check_organism" ("id" SERIAL NOT NULL, "survey_id" character varying NOT NULL, "date" TIMESTAMP NOT NULL, "organism" character varying NOT NULL, "type" character varying NOT NULL, "s1" integer NOT NULL, "s2" integer NOT NULL, "s3" integer NOT NULL, "s4" integer NOT NULL, "recorded_by" character varying, "errors" character varying, CONSTRAINT "PK_4d7d6080278505518ae5ec6cc6d" PRIMARY KEY ("id"))`,
);
await queryRunner.query(
`CREATE TABLE "reef_check_survey" ("id" character varying NOT NULL, "site_id" integer NOT NULL, "reef_check_site_id" character varying NOT NULL, "date" TIMESTAMP, "errors" character varying, "depth" double precision, "time_of_day_work_began" character varying, "time_of_day_work_ended" character varying, "method_used_to_determine_location" character varying, "river_mouth_width" character varying, "weather" character varying, "air_temp" double precision NOT NULL, "water_temp_at_surface" double precision NOT NULL, "water_temp_at3_m" double precision NOT NULL, "water_temp_at10_m" double precision NOT NULL, "approx_popn_size_x1000" double precision NOT NULL, "horizontal_visibility_in_water" double precision NOT NULL, "best_reef_area" character varying, "why_was_this_site_selected" character varying, "sheltered_or_exposed" character varying, "any_major_storms_in_last_years" character varying, "when_storms" character varying, "overall_anthro_impact" character varying, "what_kind_of_impacts" character varying, "siltation" character varying, "dynamite_fishing" character varying, "poison_fishing" character varying, "aquarium_fish_collection" character varying, "harvest_of_inverts_for_food" character varying, "harvest_of_inverts_for_curio" character varying, "tourist_diving_snorkeling" character varying, "sewage_pollution" character varying, "industrial_pollution" character varying, "commercial_fishing" character varying, "live_food_fishing" character varying, "artisinal_recreational" character varying, "other_forms_of_fishing" character varying, "other_fishing" character varying, "yachts" character varying, "level_of_other_impacts" character varying, "other_impacts" character varying, "is_site_protected" character varying, "is_protection_enforced" character varying, "level_of_poaching" character varying, "spearfishing" character varying, "banned_commercial_fishing" character varying, "recreational_fishing" character varying, "invertebrate_shell_collection" character varying, "anchoring" character varying, "diving" character varying, "other_specify" character varying, "nature_of_protection" character varying, "site_comments" character varying, "substrate_comments" character varying, "fish_comments" character varying, "inverts_comments" character varying, "comments_from_organism_sheet" character varying, "grouper_size" character varying, "percent_bleaching" character varying, "percent_colonies_bleached" character varying, "percent_of_each_colony" character varying, "suspected_disease" character varying, "rare_animals_details" character varying, "submitted_by" character varying, CONSTRAINT "PK_03c9040735ab24413d06fa712d3" PRIMARY KEY ("id"))`,
);
await queryRunner.query(
`CREATE INDEX "IDX_95fa2c48153ed166716efe3c23" ON "reef_check_survey" ("site_id") `,
);
await queryRunner.query(
`CREATE INDEX "IDX_76957dc5cc44f8ac4aac53323f" ON "reef_check_survey" ("reef_check_site_id") `,
);
await queryRunner.query(
`CREATE TABLE "reef_check_substrate" ("id" SERIAL NOT NULL, "survey_id" character varying NOT NULL, "date" TIMESTAMP NOT NULL, "substrate_code" character varying NOT NULL, "s1" integer NOT NULL, "s2" integer NOT NULL, "s3" integer NOT NULL, "s4" integer NOT NULL, "recorded_by" character varying, "errors" character varying, CONSTRAINT "PK_92e3ffa57ceea62932bbd8ab756" PRIMARY KEY ("id"))`,
);
await queryRunner.query(
`ALTER TABLE "reef_check_site" ADD CONSTRAINT "FK_228715834910cd57948e2d2dbd6" FOREIGN KEY ("site_id") REFERENCES "site"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`,
);
await queryRunner.query(
`ALTER TABLE "reef_check_organism" ADD CONSTRAINT "FK_d44cfc7847e4a645262d4d17009" FOREIGN KEY ("survey_id") REFERENCES "reef_check_survey"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`,
);
await queryRunner.query(
`ALTER TABLE "reef_check_survey" ADD CONSTRAINT "FK_95fa2c48153ed166716efe3c232" FOREIGN KEY ("site_id") REFERENCES "site"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`,
);
await queryRunner.query(
`ALTER TABLE "reef_check_survey" ADD CONSTRAINT "FK_76957dc5cc44f8ac4aac53323f3" FOREIGN KEY ("reef_check_site_id") REFERENCES "reef_check_site"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`,
);
await queryRunner.query(
`ALTER TABLE "reef_check_substrate" ADD CONSTRAINT "FK_81130cf898971ab45fea245e679" FOREIGN KEY ("survey_id") REFERENCES "reef_check_survey"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`,
);
}

public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "reef_check_substrate" DROP CONSTRAINT "FK_81130cf898971ab45fea245e679"`,
);
await queryRunner.query(
`ALTER TABLE "reef_check_survey" DROP CONSTRAINT "FK_76957dc5cc44f8ac4aac53323f3"`,
);
await queryRunner.query(
`ALTER TABLE "reef_check_survey" DROP CONSTRAINT "FK_95fa2c48153ed166716efe3c232"`,
);
await queryRunner.query(
`ALTER TABLE "reef_check_organism" DROP CONSTRAINT "FK_d44cfc7847e4a645262d4d17009"`,
);
await queryRunner.query(
`ALTER TABLE "reef_check_site" DROP CONSTRAINT "FK_228715834910cd57948e2d2dbd6"`,
);
await queryRunner.query(`DROP TABLE "reef_check_substrate"`);
await queryRunner.query(
`DROP INDEX "public"."IDX_76957dc5cc44f8ac4aac53323f"`,
);
await queryRunner.query(
`DROP INDEX "public"."IDX_95fa2c48153ed166716efe3c23"`,
);
await queryRunner.query(`DROP TABLE "reef_check_survey"`);
await queryRunner.query(`DROP TABLE "reef_check_organism"`);
await queryRunner.query(
`DROP INDEX "public"."IDX_228715834910cd57948e2d2dbd"`,
);
await queryRunner.query(`DROP TABLE "reef_check_site"`);
}
}
12 changes: 6 additions & 6 deletions packages/api/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,8 @@
"resize-survey-images": "ts-node -r dotenv/config scripts/resize-survey-images.ts",
"stormglass-csv": "ts-node -r dotenv/config scripts/generate-stormglass-csv.ts",
"fill-noaa-nearest-point": "ts-node -r dotenv/config scripts/noaa-availability.ts",
"check-video-streams": "ts-node -r dotenv/config scripts/check-video-streams.ts"
"check-video-streams": "ts-node -r dotenv/config scripts/check-video-streams.ts",
"reef-check": "ts-node -r dotenv/config scripts/reef-check.ts"
},
"dependencies": {
"@braintree/sanitize-url": "^7.0.1",
Expand All @@ -71,21 +72,20 @@
"@nestjs/common": "^9.3.9",
"@nestjs/core": "^9.3.9",
"@nestjs/passport": "^9.0.3",
"@nestjs/platform-express": "^10.4.5",
"@nestjs/platform-express": "^10.4.14",
"@nestjs/schedule": "^3.0.3",
"@nestjs/swagger": "^6.3.0",
"@nestjs/typeorm": "^9.0.1",
"@turf/turf": "^6.5.0",
"@types/bluebird": "^3.5.32",
"@types/node-xlsx": "^0.15.3",
"axios": "^1.7.8",
"axios": "^1.7.4",
"axios-retry": "^3.8.1",
"bluebird": "^3.7.2",
"class-transformer": "^0.3.1",
"class-validator": "^0.14.0",
"csv-parse": "^4.15.1",
"csv-stringify": "^6.4.0",
"express": "^4.21.1",
"express": "^4.21.2",
"firebase-admin": "^12.0.0",
"geo-tz": "^6.0.0",
"geojson": "^0.5.0",
Expand All @@ -95,7 +95,7 @@
"luxon": "^3.3.0",
"md5-file": "^5.0.0",
"multer": "^1.4.2",
"node-xlsx": "^0.17.2",
"node-xlsx": "^0.24.0",
"objects-to-csv": "^1.3.6",
"passport": "^0.6.0",
"passport-custom": "^1.1.1",
Expand Down
Loading

0 comments on commit ca5e4b9

Please sign in to comment.