From 5208f9d70e2e495fcf58ad001611688de76111fb Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Fri, 29 Mar 2024 17:28:21 -0400 Subject: [PATCH 01/23] WORK IN PROGRESS --- package.json | 8 +++++++ .../intersect-polygon.module.js | 22 +++++++++++++------ .../intersect-polygon.test.js | 22 +++++++++++++++++-- src/max/index.js | 2 +- src/max/max.test.js | 16 +++++++++++++- src/mean/index.js | 2 +- src/mean/mean.test.js | 16 +++++++++++++- src/median/index.js | 2 +- src/median/median.test.js | 14 ++++++++++++ src/min/index.js | 2 +- src/min/min.test.js | 16 +++++++++++++- src/mode/index.js | 2 +- src/mode/mode.test.js | 16 +++++++++++++- src/modes/index.js | 2 +- src/modes/modes.test.js | 14 ++++++++++++ src/range/index.js | 2 +- src/range/range.test.js | 16 +++++++++++++- src/stat/stat.module.js | 4 ++-- src/stats/stats.core.js | 19 ++++++++++++++-- src/sum/index.js | 4 ++-- src/sum/sum.test.js | 10 +++++++++ 21 files changed, 184 insertions(+), 27 deletions(-) diff --git a/package.json b/package.json index 80dc6b0..d7424eb 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,14 @@ "lint": "eslint lite && eslint src", "serve": "npx srvd --debug --port=3000 --wait=120", "test": "set -e; for f in src/*/*test*.js; do echo \"\nrunning $f\" && sleep 5 && node -r esm $f; done", + "test:intersect-polygon": "node -r esm ./src/intersect-polygon/intersect-polygon.test.js", + "test:max": "node -r esm ./src/max/max.test.js", + "test:mean": "node -r esm ./src/mean/mean.test.js", + "test:median": "node -r esm ./src/median/median.test.js", + "test:min": "node -r esm ./src/min/min.test.js", + "test:mode": "node -r esm ./src/mode/mode.test.js", + "test:range": "node -r esm ./src/range/range.test.js", + "test:sum": "node -r esm ./src/sum/sum.test.js", "test-loading-builds": "node test-loading-builds.js", "setup": "bash setup.sh" }, diff --git a/src/intersect-polygon/intersect-polygon.module.js b/src/intersect-polygon/intersect-polygon.module.js index 9b8aef6..0183f76 100644 --- a/src/intersect-polygon/intersect-polygon.module.js +++ b/src/intersect-polygon/intersect-polygon.module.js @@ -11,18 +11,23 @@ import snap from "snap-bbox"; import wrapParse from "../wrap-parse"; // import writePng from "@danieljdufour/write-png"; -const intersectPolygon = (georaster, geometry, perPixelFunction, { debug_level = 0 } = {}) => { +const intersectPolygon = (georaster, geometry, perPixelFunction, { debug_level = 0, vrm = [1, 1] } = {}) => { const georaster_bbox = [georaster.xmin, georaster.ymin, georaster.xmax, georaster.ymax]; const precisePixelHeight = georaster.pixelHeight.toString(); const precisePixelWidth = georaster.pixelWidth.toString(); + if (typeof vrm === "number") vrm = [vrm, vrm] + const [xvrm, yvrm] = vrm; + console.log({xvrm, yvrm}); + // run intersect for each sample // each sample is a multi-dimensional array of numbers // in the following xdim format [b][r][c] let samples; if (georaster.values) { + console.log("georaster.values:", georaster.values); // if we have already loaded all the values into memory, // just pass those along and avoid using up more memory const sample = georaster.values; @@ -113,15 +118,18 @@ const intersectPolygon = (georaster, geometry, perPixelFunction, { debug_level = const sample_bbox = precise_sample_bbox.map(str => Number(str)); + console.log({vrm}); + console.log(JSON.stringify(geometry)); const intersect_params = { debug: true, raster_bbox: sample_bbox, - raster_height: sample_height, - raster_width: sample_width, - pixel_height: georaster.pixelHeight, - pixel_width: georaster.pixelWidth, + raster_height: sample_height * yvrm, + raster_width: sample_width * xvrm, + pixel_height: georaster.pixelHeight / yvrm, + pixel_width: georaster.pixelWidth / xvrm, geometry }; + console.log("intersect_params:", intersect_params); const intersections = dufour_peyton_intersection.calculate(intersect_params); if (debug_level >= 3) console.log("[geoblaze] intersections:", JSON.stringify(intersections, undefined, 2)); @@ -145,9 +153,9 @@ const intersectPolygon = (georaster, geometry, perPixelFunction, { debug_level = row.forEach(([start, end], irange) => { for (let icol = start; icol <= end; icol++) { imageBands.forEach((band, iband) => { - const row = band[irow]; + const row = band[Math.floor(irow / yvrm)]; if (row) { - const value = row[icol]; + const value = row[Math.floor(icol / xvrm)]; perPixelFunction(value, iband, yoff + irow, xoff + icol); } }); diff --git a/src/intersect-polygon/intersect-polygon.test.js b/src/intersect-polygon/intersect-polygon.test.js index 247e295..96d1df5 100644 --- a/src/intersect-polygon/intersect-polygon.test.js +++ b/src/intersect-polygon/intersect-polygon.test.js @@ -11,13 +11,13 @@ import parse from "../parse"; import { convertMultiPolygon } from "../convert-geometry"; import intersectPolygon from "../intersect-polygon"; -async function countIntersectingPixels(georaster, geom, includeNoData) { +async function countIntersectingPixels(georaster, geom, includeNoData, { debug_level, vrm } = {}) { let numberOfIntersectingPixels = 0; await intersectPolygon(georaster, geom, value => { if (includeNoData || value !== georaster.noDataValue) { numberOfIntersectingPixels++; } - }); + }, { debug_level, vrm }); return numberOfIntersectingPixels; } @@ -197,3 +197,21 @@ test("more testing", async ({ eq }) => { // same as rasterstats eq(numberOfIntersectingPixels, 1_672); }); + +test("virtual resampling", async ({ eq }) => { + const georaster = await parse(urlToData + "/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"); + const geojson = await fetch_json(urlToData + "/virtual-resampling/virtual-resampling-one.geojson"); + const geom = convertMultiPolygon(geojson); + const numberOfIntersectingPixels = await countIntersectingPixels(georaster, geom, false, { debug_level: 10, vrm: 100 }); + // same as rasterstats + eq(numberOfIntersectingPixels, 104); +}) + +test("virtual resampling intersect", async ({ eq }) => { + const georaster = await parse(urlToData + "/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"); + const geojson = await fetch_json(urlToData + "/virtual-resampling/virtual-resampling-intersect.geojson"); + const geom = convertMultiPolygon(geojson); + const numberOfIntersectingPixels = await countIntersectingPixels(georaster, geom, false, { debug_level: 10, vrm: 100 }); + // same as rasterstats + eq(numberOfIntersectingPixels, 1577); +}) diff --git a/src/max/index.js b/src/max/index.js index 4ef0bf5..995a11d 100644 --- a/src/max/index.js +++ b/src/max/index.js @@ -19,5 +19,5 @@ import stat from "../stat"; */ export default function max(georaster, geometry, test) { - return stat(georaster, geometry, "max", test); + return stat(georaster, geometry, "max", test, { vrm: 100 }); } diff --git a/src/max/max.test.js b/src/max/max.test.js index 826fa23..d4c1ac7 100644 --- a/src/max/max.test.js +++ b/src/max/max.test.js @@ -5,7 +5,7 @@ import { serve } from "srvd"; import load from "../load"; import max from "."; -serve({ debug: true, max: 1, port: 3000 }); +serve({ debug: true, max: 5, port: 3000 }); const url = "http://localhost:3000/data/test.tiff"; @@ -61,3 +61,17 @@ test("Max with Web Mercator Bounding Box and GeoRaster URL", async ({ eq }) => { const value = Number(result[0].toFixed(2)); eq(value, expectedBboxValue); }); + +test("virtual resampling, contained", async ({ eq }) => { + const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; + const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-one.geojson").then(res => res.json()); + const result = await max(url, geojson) + eq(result, [38]); +}) + +test("virtual resampling, intersecting 4 pixels", async ({ eq }) => { + const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; + const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-intersect.geojson").then(res => res.json()); + const result = await max(url, geojson) + eq(result, [38]); +}) diff --git a/src/mean/index.js b/src/mean/index.js index ae2912e..c484004 100644 --- a/src/mean/index.js +++ b/src/mean/index.js @@ -19,5 +19,5 @@ import stat from "../stat"; */ export default function mean(georaster, geometry, test) { - return stat(georaster, geometry, "mean", test); + return stat(georaster, geometry, "mean", test, { vrm: 100 }); } diff --git a/src/mean/mean.test.js b/src/mean/mean.test.js index 7c9907d..ed552aa 100644 --- a/src/mean/mean.test.js +++ b/src/mean/mean.test.js @@ -5,7 +5,7 @@ import load from "../load"; import mean from "."; import utils from "../utils"; -serve({ debug: true, max: 2, port: 3000 }); +serve({ debug: true, max: 4, port: 3000 }); const { round } = utils; @@ -91,3 +91,17 @@ test("(Modern) Mean from GeoJSON", async ({ eq }) => { const value = round(results[0]); eq(value, expectedPolygonGeojsonValue); }); + +test("virtual resampling, contained", async ({ eq }) => { + const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; + const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-one.geojson").then(res => res.json()); + const result = await mean(url, geojson) + eq(result, [38]); +}) + +test("virtual resampling, intersecting 4 pixels", async ({ eq }) => { + const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; + const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-intersect.geojson").then(res => res.json()); + const result = await mean(url, geojson) + eq(result, [33.61065313887127]); +}) \ No newline at end of file diff --git a/src/median/index.js b/src/median/index.js index 7fbd4a6..1e29289 100644 --- a/src/median/index.js +++ b/src/median/index.js @@ -19,5 +19,5 @@ import stat from "../stat"; */ export default function median(georaster, geometry, test) { - return stat(georaster, geometry, "median", test); + return stat(georaster, geometry, "median", test, { vrm: 100 }); } diff --git a/src/median/median.test.js b/src/median/median.test.js index d830ceb..48e3625 100644 --- a/src/median/median.test.js +++ b/src/median/median.test.js @@ -78,3 +78,17 @@ test("Get Median from Whole Raster from url", async ({ eq }) => { const value = result[0]; eq(value, 0); }); + +test("virtual resampling, contained", async ({ eq }) => { + const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; + const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-one.geojson").then(res => res.json()); + const result = await median(url, geojson) + eq(result, [38]); +}) + +test("virtual resampling, intersecting 4 pixels", async ({ eq }) => { + const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; + const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-intersect.geojson").then(res => res.json()); + const result = await median(url, geojson) + eq(result, [38]); +}) diff --git a/src/min/index.js b/src/min/index.js index 8449dea..9fd4ecc 100644 --- a/src/min/index.js +++ b/src/min/index.js @@ -19,5 +19,5 @@ import stat from "../stat"; */ export default function min(georaster, geometry, test) { - return stat(georaster, geometry, "min", test); + return stat(georaster, geometry, "min", test, { vrm: 100 }); } diff --git a/src/min/min.test.js b/src/min/min.test.js index e87f53a..f3e2e73 100644 --- a/src/min/min.test.js +++ b/src/min/min.test.js @@ -3,7 +3,7 @@ import { serve } from "srvd"; import load from "../load"; import min from "."; -serve({ debug: true, max: 1, port: 3000 }); +serve({ debug: true, max: 5, port: 3000 }); const url = "http://localhost:3000/data/test.tiff"; const bbox = [80.63, 7.42, 84.21, 10.1]; @@ -61,3 +61,17 @@ test("(Modern) Get Min from whole Raster", async ({ eq }) => { const results = await min(url); eq(results, [expectedPolygonValue]); }); + +test("virtual resampling, contained", async ({ eq }) => { + const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; + const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-one.geojson").then(res => res.json()); + const result = await min(url, geojson) + eq(result, [38]); +}) + +test("virtual resampling, intersecting 4 pixels", async ({ eq }) => { + const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; + const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-intersect.geojson").then(res => res.json()); + const result = await min(url, geojson) + eq(result, [1]); +}) diff --git a/src/mode/index.js b/src/mode/index.js index 88b5aed..80b4059 100644 --- a/src/mode/index.js +++ b/src/mode/index.js @@ -20,5 +20,5 @@ import stat from "../stat"; */ export default function mode(georaster, geometry, test) { - return stat(georaster, geometry, "mode", test); + return stat(georaster, geometry, "mode", test, { vrm: 100 }); } diff --git a/src/mode/mode.test.js b/src/mode/mode.test.js index 9c568fd..9adead1 100644 --- a/src/mode/mode.test.js +++ b/src/mode/mode.test.js @@ -4,7 +4,7 @@ import { serve } from "srvd"; import load from "../load"; import mode from "."; -serve({ debug: true, max: 1, port: 3000 }); +serve({ debug: true, max: 5, port: 3000 }); const url = "http://localhost:3000/data/test.tiff"; @@ -44,3 +44,17 @@ test("(Modern) Mode Polygon", async ({ eq }) => { const results = await mode(url, polygon); eq(results, [0]); }); + +test("virtual resampling, contained", async ({ eq }) => { + const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; + const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-one.geojson").then(res => res.json()); + const result = await mode(url, geojson) + eq(result, [38]); +}) + +test("virtual resampling, intersecting 4 pixels", async ({ eq }) => { + const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; + const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-intersect.geojson").then(res => res.json()); + const result = await mode(url, geojson) + eq(result, [38]); +}) diff --git a/src/modes/index.js b/src/modes/index.js index 448de33..920efe7 100644 --- a/src/modes/index.js +++ b/src/modes/index.js @@ -20,5 +20,5 @@ import stat from "../stat"; */ export default function modes(georaster, geometry, test) { - return stat(georaster, geometry, "modes", test); + return stat(georaster, geometry, "modes", test, { vrm: 100 }); } diff --git a/src/modes/modes.test.js b/src/modes/modes.test.js index d355d05..eed02e1 100644 --- a/src/modes/modes.test.js +++ b/src/modes/modes.test.js @@ -44,3 +44,17 @@ test("(Modern) Modes Polygon", async ({ eq }) => { const results = await modes(url, polygon); eq(results, [[0]]); }); + +test("virtual resampling, contained", async ({ eq }) => { + const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; + const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-one.geojson").then(res => res.json()); + const result = await modes(url, geojson) + eq(result, [38]); +}) + +test("virtual resampling, intersecting 4 pixels", async ({ eq }) => { + const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; + const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-intersect.geojson").then(res => res.json()); + const result = await modes(url, geojson) + eq(result, [38]); +}) \ No newline at end of file diff --git a/src/range/index.js b/src/range/index.js index 56a4894..37b6e66 100644 --- a/src/range/index.js +++ b/src/range/index.js @@ -19,5 +19,5 @@ import stat from "../stat"; */ export default function range(georaster, geometry, test) { - return stat(georaster, geometry, "range", test); + return stat(georaster, geometry, "range", test, { vrm: 100 }); } diff --git a/src/range/range.test.js b/src/range/range.test.js index 608aeb7..96ca0bf 100644 --- a/src/range/range.test.js +++ b/src/range/range.test.js @@ -2,7 +2,7 @@ import test from "flug"; import { serve } from "srvd"; import range from "."; -serve({ debug: true, max: 20, port: 3000 }); +serve({ debug: true, max: 25, port: 3000 }); const url = "http://localhost:3000/data/test.tiff"; const bbox = [80.63, 7.42, 84.21, 10.1]; @@ -41,3 +41,17 @@ test("(Modern) Get Range from whole Raster", async ({ eq }) => { const results = await range(url); eq(results, [8131.2]); }); + +test("virtual resampling, contained", async ({ eq }) => { + const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; + const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-one.geojson").then(res => res.json()); + const result = await range(url, geojson) + eq(result, [0]); +}) + +test("virtual resampling, intersecting 4 pixels", async ({ eq }) => { + const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; + const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-intersect.geojson").then(res => res.json()); + const result = await range(url, geojson) + eq(result, [37]); +}) diff --git a/src/stat/stat.module.js b/src/stat/stat.module.js index 1b2db34..4c87243 100644 --- a/src/stat/stat.module.js +++ b/src/stat/stat.module.js @@ -1,6 +1,6 @@ import QuickPromise from "quick-promise"; import stats from "../stats"; -export default function stat(georaster, geometry, stat, test) { - return QuickPromise.resolve(stats(georaster, geometry, { stats: [stat] }, test)).then(stats => stats.map(it => it[stat])); +export default function stat(georaster, geometry, stat, test, options) { + return QuickPromise.resolve(stats(georaster, geometry, { stats: [stat] }, test, options)).then(stats => stats.map(it => it[stat])); } diff --git a/src/stats/stats.core.js b/src/stats/stats.core.js index 6103d64..f235f3a 100644 --- a/src/stats/stats.core.js +++ b/src/stats/stats.core.js @@ -6,7 +6,7 @@ import wrap from "../wrap-parse"; import { convertBbox, convertMultiPolygon } from "../convert-geometry"; import intersectPolygon from "../intersect-polygon"; -const stats = (georaster, geometry, calcStatsOptions, test, { debug_level = 0 } = {}) => { +const stats = (georaster, geometry, calcStatsOptions, test, { debug_level = 0, vrm } = {}) => { try { // shallow clone calcStatsOptions = { ...calcStatsOptions }; @@ -18,6 +18,20 @@ const stats = (georaster, geometry, calcStatsOptions, test, { debug_level = 0 } calcStatsOptions.noData = noDataValue; } + let xvrm; + let yvrm; + if (typeof vrm === "number") { + if (vrm !== Math.round(vrm)) { + throw new Error("[geoblaze] divisor must be an integer"); + } + xvrm = vrm; + yvrm = vrm; + } else if (Array.isArray(vrm) && vrm.length === 2 && typeof vrm[0] === "number") { + [xvrm, yvrm] = vrm; + } + + console.log("vrm:", [xvrm, yvrm]); + if (test) { if (calcStatsOptions && calcStatsOptions.filter) { const original_filter = calcStatsOptions.filter; @@ -37,6 +51,7 @@ const stats = (georaster, geometry, calcStatsOptions, test, { debug_level = 0 } } else if (utils.isBbox(geometry)) { if (debug_level >= 2) console.log("[geoblaze] geometry is a rectangle"); geometry = convertBbox(geometry); + // if using multiplier, might need to pad get results or at least not round const values = get(georaster, geometry, flat); return QuickPromise.resolve(values).then(getStatsByBand); } else if (utils.isPolygonal(geometry)) { @@ -59,7 +74,7 @@ const stats = (georaster, geometry, calcStatsOptions, test, { debug_level = 0 } values[bandIndex] = [value]; } }, - { debug_level } + { debug_level, vrm } ); return QuickPromise.resolve(done).then(() => { diff --git a/src/sum/index.js b/src/sum/index.js index 38f3405..7eebc35 100644 --- a/src/sum/index.js +++ b/src/sum/index.js @@ -17,6 +17,6 @@ import stat from "../stat"; * // results is [red, green, blue, nir] * [217461, 21375, 57312, 457125] */ -export default function sum(georaster, geometry, test) { - return stat(georaster, geometry, "sum", test); +export default function sum(georaster, geometry, test, { vrm } = {}) { + return stat(georaster, geometry, "sum", test, { vrm }); } diff --git a/src/sum/sum.test.js b/src/sum/sum.test.js index a9e85c2..24db67e 100644 --- a/src/sum/sum.test.js +++ b/src/sum/sum.test.js @@ -305,3 +305,13 @@ test("(Modern) Get Sum from Polygon Above X Value", async ({ eq }) => { const value = Number(sum(georaster, polygon, v => v > 3000)[0].toFixed(2)); eq(value, 1_454_066); }); + +test("Virtual Resampling", async ({ eq }) => { + const values = [ + await load(`http://localhost:${port}/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif`), + await fetchJson(`http://localhost:${port}/data/virtual-resampling/virtual-resampling-one.geojson`) + ]; + const [georaster, geojson] = values; + const results = await sum(georaster, geojson, undefined, { vrm: 10 }); + eq(results, []); +}); From 5191d49daad38f5db3137357e0f5b52405744e0a Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Sun, 14 Apr 2024 20:09:03 -0500 Subject: [PATCH 02/23] added modes to the package.json --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index d7424eb..a235f53 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "test:median": "node -r esm ./src/median/median.test.js", "test:min": "node -r esm ./src/min/min.test.js", "test:mode": "node -r esm ./src/mode/mode.test.js", + "test:modes": "node -r esm ./src/modes/modes.test.js", "test:range": "node -r esm ./src/range/range.test.js", "test:sum": "node -r esm ./src/sum/sum.test.js", "test-loading-builds": "node test-loading-builds.js", From 4fcae791737e80d1710559ed5a08128e73912903 Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Sun, 28 Apr 2024 19:27:42 -0400 Subject: [PATCH 03/23] refactored intersect-polygon --- .../intersect-polygon.module.js | 66 +++++++++++----- .../intersect-polygon.test.js | 77 +++++++++++++++---- 2 files changed, 108 insertions(+), 35 deletions(-) diff --git a/src/intersect-polygon/intersect-polygon.module.js b/src/intersect-polygon/intersect-polygon.module.js index 0183f76..5c810c2 100644 --- a/src/intersect-polygon/intersect-polygon.module.js +++ b/src/intersect-polygon/intersect-polygon.module.js @@ -1,7 +1,9 @@ import bboxArea from "bbox-fns/bbox-area.js"; +import bboxSize from "bbox-fns/bbox-size.js"; import booleanIntersects from "bbox-fns/boolean-intersects.js"; import calcAll from "bbox-fns/calc-all.js"; import dufour_peyton_intersection from "dufour-peyton-intersection"; +import fastMax from "fast-max"; import merge from "bbox-fns/merge.js"; import union from "bbox-fns/union.js"; import reproject from "bbox-fns/precise/reproject.js"; @@ -11,15 +13,55 @@ import snap from "snap-bbox"; import wrapParse from "../wrap-parse"; // import writePng from "@danieljdufour/write-png"; -const intersectPolygon = (georaster, geometry, perPixelFunction, { debug_level = 0, vrm = [1, 1] } = {}) => { +const VRM_NO_RESAMPLING = [1, 1]; + +const intersectPolygon = (georaster, geometry, perPixelFunction, { debug_level = 0, vrm = VRM_NO_RESAMPLING } = {}) => { const georaster_bbox = [georaster.xmin, georaster.ymin, georaster.xmax, georaster.ymax]; const precisePixelHeight = georaster.pixelHeight.toString(); const precisePixelWidth = georaster.pixelWidth.toString(); - if (typeof vrm === "number") vrm = [vrm, vrm] + if (typeof vrm === "number") { + if (vrm <= 0 || vrm !== Math.round(vrm)) { + throw new Error("[geoblaze] vrm can only be defined as a positive integer"); + } + vrm = [vrm, vrm]; + } + + let geometry_bboxes = union(calcAll(geometry)); + if (debug_level >= 2) console.log("[geoblaze] geometry_bboxes:", geometry_bboxes); + + // filter out geometry bboxes that don't intersect raster + // assuming anti-meridian geometries have been normalized beforehand + geometry_bboxes = geometry_bboxes.filter(bbox => booleanIntersects(bbox, georaster_bbox)); + if (debug_level >= 2) console.log("[geoblaze] intersecting geometry bboxes:", geometry_bboxes); + + // no intersecting pixels + if (geometry_bboxes.length === 0) return; + + // calculate size of bboxes in pixels + const geometry_bboxes_sizes = geometry_bboxes.map(bbox => bboxSize(bbox)); + if (debug_level >= 2) console.log("[geoblaze] size of geometry bboxes:", geometry_bboxes_sizes); + + const geometry_bbox_size_ratios = geometry_bboxes_sizes.map(([width, height]) => [width / georaster.pixelWidth, height / georaster.pixelHeight]); + if (debug_level >= 2) console.log("[geoblaze] geometry_bbox_size_ratios:", geometry_bbox_size_ratios); + + // resample just enough to ensure at least one resampled pixel intersects + if (vrm === "minimal") { + // check if any geometry is smaller than half a pixel + if (geometry_bbox_size_ratios.some(([xratio, yratio]) => xratio <= 1 || yratio <= 1)) { + const geometry_bboxes_multipliers = geometry_bbox_size_ratios.map(([xratio, yratio]) => [2 / xratio, 2 / yratio]); + vrm = [ + Math.ceil(fastMax(geometry_bboxes_multipliers.map(([xmul, ymul]) => xmul))), + Math.ceil(fastMax(geometry_bboxes_multipliers.map(([xmul, ymul]) => ymul))) + ]; + } else { + vrm = VRM_NO_RESAMPLING; + } + } + + if (debug_level >= 2) console.log("[geoblaze] vrm:", vrm); const [xvrm, yvrm] = vrm; - console.log({xvrm, yvrm}); // run intersect for each sample // each sample is a multi-dimensional array of numbers @@ -27,7 +69,7 @@ const intersectPolygon = (georaster, geometry, perPixelFunction, { debug_level = let samples; if (georaster.values) { - console.log("georaster.values:", georaster.values); + if (debug_level >= 2) console.log("[geoblaze] georaster already has all values loaded into memory"); // if we have already loaded all the values into memory, // just pass those along and avoid using up more memory const sample = georaster.values; @@ -46,17 +88,6 @@ const intersectPolygon = (georaster, geometry, perPixelFunction, { debug_level = samples = [{ intersections, sample, offset: [0, 0] }]; } else if (georaster.getValues) { - let geometry_bboxes = union(calcAll(geometry)); - if (debug_level >= 2) console.log("[geoblaze] geometry_bboxes:", geometry_bboxes); - - // filter out geometry bboxes that don't intersect raster - // assuming anti-meridian geometries have been normalized beforehand - geometry_bboxes = geometry_bboxes.filter(bbox => booleanIntersects(bbox, georaster_bbox)); - if (debug_level >= 2) console.log("[geoblaze] intersecting geometry bboxes:", geometry_bboxes); - - // no intersecting pixels - if (geometry_bboxes.length === 0) return; - const geotransfrom = PreciseGeotransform([georaster.xmin.toString(), precisePixelWidth, "0", georaster.ymax.toString(), "0", "-" + precisePixelHeight]); if (debug_level >= 2) console.log("[geoblaze] geotransfrom:", geotransfrom); @@ -118,8 +149,6 @@ const intersectPolygon = (georaster, geometry, perPixelFunction, { debug_level = const sample_bbox = precise_sample_bbox.map(str => Number(str)); - console.log({vrm}); - console.log(JSON.stringify(geometry)); const intersect_params = { debug: true, raster_bbox: sample_bbox, @@ -129,7 +158,7 @@ const intersectPolygon = (georaster, geometry, perPixelFunction, { debug_level = pixel_width: georaster.pixelWidth / xvrm, geometry }; - console.log("intersect_params:", intersect_params); + if (debug_level >= 3) console.log("[geoblaze] intersect_params:", intersect_params); const intersections = dufour_peyton_intersection.calculate(intersect_params); if (debug_level >= 3) console.log("[geoblaze] intersections:", JSON.stringify(intersections, undefined, 2)); @@ -164,6 +193,7 @@ const intersectPolygon = (georaster, geometry, perPixelFunction, { debug_level = }); }); }); + return { vrm }; }); }; diff --git a/src/intersect-polygon/intersect-polygon.test.js b/src/intersect-polygon/intersect-polygon.test.js index 96d1df5..d7ee1d4 100644 --- a/src/intersect-polygon/intersect-polygon.test.js +++ b/src/intersect-polygon/intersect-polygon.test.js @@ -13,12 +13,17 @@ import intersectPolygon from "../intersect-polygon"; async function countIntersectingPixels(georaster, geom, includeNoData, { debug_level, vrm } = {}) { let numberOfIntersectingPixels = 0; - await intersectPolygon(georaster, geom, value => { - if (includeNoData || value !== georaster.noDataValue) { - numberOfIntersectingPixels++; - } - }, { debug_level, vrm }); - return numberOfIntersectingPixels; + const result = await intersectPolygon( + georaster, + geom, + value => { + if (includeNoData || value !== georaster.noDataValue) { + numberOfIntersectingPixels++; + } + }, + { debug_level, vrm } + ); + return { numberOfIntersectingPixels, ...result }; } async function fetch_json(url) { @@ -70,7 +75,7 @@ test("(Legacy) Testing intersection of box", async ({ eq }) => { georaster.ymax - 0.5 * pixelHeight ]); const coords = geom.geometry.coordinates; - const numberOfIntersectingPixels = await countIntersectingPixels(georaster, coords, false); + const { numberOfIntersectingPixels } = await countIntersectingPixels(georaster, coords, false); eq(numberOfIntersectingPixels, expectedNumberOfIntersectingPixels); }); @@ -79,7 +84,7 @@ test("(Legacy) Test intersection/sum calculations for Country with Multiple Ring const response = await fetch(urlToGeojson); const country = await response.json(); const geom = convertMultiPolygon(country); - const numberOfIntersectingPixels = await countIntersectingPixels(georaster, geom, true); + const { numberOfIntersectingPixels } = await countIntersectingPixels(georaster, geom, true); eq(numberOfIntersectingPixels, EXPECTED_NUMBER_OF_INTERSECTING_PIXELS); }); @@ -168,6 +173,26 @@ test("antimerdian #1", async ({ eq }) => { eq(numberOfIntersectingPixels, 314_930); }); +test("antimerdian #1, vrm=10", async ({ eq }) => { + const georaster = await parse(urlToData + "gfwfiji_6933_COG.tiff"); + let geojson = await fetch_json(urlToData + "antimeridian/right-edge.geojson"); + let numberOfIntersectingPixels = 0; + // reproject geometry to projection of raster + geojson = reprojectGeoJSON(geojson, { from: 4326, to: georaster.projection }); + const geom = convertMultiPolygon(geojson); + await intersectPolygon( + georaster, + geom, + value => { + if (value !== georaster.noDataValue) { + numberOfIntersectingPixels++; + } + }, + { vrm: 10 } + ); + eq(numberOfIntersectingPixels, 31_501_375); +}); + test("parse", async ({ eq }) => { const georaster = await parse(urlToData + "geotiff-test-data/gfw-azores.tif"); const geojson = await fetch_json(urlToData + "santa-maria/santa-maria-mpa.geojson"); @@ -193,25 +218,43 @@ test("more testing", async ({ eq }) => { const georaster = await parse(urlToData + "test.tiff"); const geojson = await fetch_json(urlToData + "part-of-india.geojson"); const geom = convertMultiPolygon(geojson); - const numberOfIntersectingPixels = await countIntersectingPixels(georaster, geom, false); + const { numberOfIntersectingPixels } = await countIntersectingPixels(georaster, geom, false); // same as rasterstats eq(numberOfIntersectingPixels, 1_672); }); -test("virtual resampling", async ({ eq }) => { +test("virtual resampling with vrm = minimal", async ({ eq }) => { const georaster = await parse(urlToData + "/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"); const geojson = await fetch_json(urlToData + "/virtual-resampling/virtual-resampling-one.geojson"); const geom = convertMultiPolygon(geojson); - const numberOfIntersectingPixels = await countIntersectingPixels(georaster, geom, false, { debug_level: 10, vrm: 100 }); - // same as rasterstats + const { numberOfIntersectingPixels, vrm } = await countIntersectingPixels(georaster, geom, false, { debug_level: 0, vrm: "minimal" }); + eq(vrm, [12, 21]); + eq(numberOfIntersectingPixels, 2); +}); + +test("virtual resampling with vrm = 100", async ({ eq }) => { + const georaster = await parse(urlToData + "/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"); + const geojson = await fetch_json(urlToData + "/virtual-resampling/virtual-resampling-one.geojson"); + const geom = convertMultiPolygon(geojson); + const { numberOfIntersectingPixels, vrm } = await countIntersectingPixels(georaster, geom, false, { debug_level: 0, vrm: 100 }); + eq(vrm, [100, 100]); eq(numberOfIntersectingPixels, 104); -}) +}); -test("virtual resampling intersect", async ({ eq }) => { +test("virtual resampling intersect with vrm = minimal", async ({ eq }) => { const georaster = await parse(urlToData + "/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"); const geojson = await fetch_json(urlToData + "/virtual-resampling/virtual-resampling-intersect.geojson"); const geom = convertMultiPolygon(geojson); - const numberOfIntersectingPixels = await countIntersectingPixels(georaster, geom, false, { debug_level: 10, vrm: 100 }); - // same as rasterstats + const { numberOfIntersectingPixels, vrm } = await countIntersectingPixels(georaster, geom, false, { debug_level: 0, vrm: "minimal" }); + eq(vrm, [4, 4]); + eq(numberOfIntersectingPixels, 2); +}); + +test("virtual resampling intersect with vrm = 100", async ({ eq }) => { + const georaster = await parse(urlToData + "/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"); + const geojson = await fetch_json(urlToData + "/virtual-resampling/virtual-resampling-intersect.geojson"); + const geom = convertMultiPolygon(geojson); + const { numberOfIntersectingPixels, vrm } = await countIntersectingPixels(georaster, geom, false, { debug_level: 0, vrm: 100 }); + eq(vrm, [100, 100]); eq(numberOfIntersectingPixels, 1577); -}) +}); From d4a1d3915e2bf05fecd3dc98709798849ddaccf9 Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Sat, 11 May 2024 09:40:56 -0400 Subject: [PATCH 04/23] cleaned up more virtual resampling code and added resampling support to georasters fully loaded into memory --- package.json | 1 + .../intersect-polygon.module.js | 13 +- src/max/index.js | 2 +- src/max/max.test.js | 14 +-- src/mean/index.js | 2 +- src/mean/mean.test.js | 27 +++- src/median/index.js | 2 +- src/median/median.test.js | 12 +- src/min/index.js | 2 +- src/min/min.test.js | 14 +-- src/mode/index.js | 2 +- src/mode/mode.test.js | 12 +- src/modes/index.js | 2 +- src/modes/modes.test.js | 12 +- src/range/index.js | 2 +- src/range/range.test.js | 22 ++-- src/stats/stats.core.js | 116 +++++++++++++++--- src/stats/stats.test.js | 65 +++++++++- src/sum/index.js | 5 +- src/sum/sum.test.js | 4 +- 20 files changed, 250 insertions(+), 81 deletions(-) diff --git a/package.json b/package.json index a235f53..28393ca 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "test:mode": "node -r esm ./src/mode/mode.test.js", "test:modes": "node -r esm ./src/modes/modes.test.js", "test:range": "node -r esm ./src/range/range.test.js", + "test:stats": "node -r esm ./src/stats/stats.test.js", "test:sum": "node -r esm ./src/sum/sum.test.js", "test-loading-builds": "node test-loading-builds.js", "setup": "bash setup.sh" diff --git a/src/intersect-polygon/intersect-polygon.module.js b/src/intersect-polygon/intersect-polygon.module.js index 5c810c2..e702130 100644 --- a/src/intersect-polygon/intersect-polygon.module.js +++ b/src/intersect-polygon/intersect-polygon.module.js @@ -52,8 +52,9 @@ const intersectPolygon = (georaster, geometry, perPixelFunction, { debug_level = if (geometry_bbox_size_ratios.some(([xratio, yratio]) => xratio <= 1 || yratio <= 1)) { const geometry_bboxes_multipliers = geometry_bbox_size_ratios.map(([xratio, yratio]) => [2 / xratio, 2 / yratio]); vrm = [ - Math.ceil(fastMax(geometry_bboxes_multipliers.map(([xmul, ymul]) => xmul))), - Math.ceil(fastMax(geometry_bboxes_multipliers.map(([xmul, ymul]) => ymul))) + // don't drop more than 10,000 sample lines per pixel + Math.min(10000, Math.ceil(fastMax(geometry_bboxes_multipliers.map(([xmul, ymul]) => xmul)))), + Math.min(10000, Math.ceil(fastMax(geometry_bboxes_multipliers.map(([xmul, ymul]) => ymul)))) ]; } else { vrm = VRM_NO_RESAMPLING; @@ -79,10 +80,10 @@ const intersectPolygon = (georaster, geometry, perPixelFunction, { debug_level = const intersections = dufour_peyton_intersection.calculate({ debug: false, raster_bbox: georaster_bbox, - raster_height: georaster.height, - raster_width: georaster.width, - pixel_height: georaster.pixelHeight, - pixel_width: georaster.pixelWidth, + raster_height: georaster.height * yvrm, + raster_width: georaster.width * xvrm, + pixel_height: georaster.pixelHeight / yvrm, + pixel_width: georaster.pixelWidth / xvrm, geometry }); diff --git a/src/max/index.js b/src/max/index.js index 995a11d..b127dec 100644 --- a/src/max/index.js +++ b/src/max/index.js @@ -19,5 +19,5 @@ import stat from "../stat"; */ export default function max(georaster, geometry, test) { - return stat(georaster, geometry, "max", test, { vrm: 100 }); + return stat(georaster, geometry, "max", test, { vrm: "minimal" }); } diff --git a/src/max/max.test.js b/src/max/max.test.js index d4c1ac7..c76aaac 100644 --- a/src/max/max.test.js +++ b/src/max/max.test.js @@ -5,7 +5,7 @@ import { serve } from "srvd"; import load from "../load"; import max from "."; -serve({ debug: true, max: 5, port: 3000 }); +serve({ debug: true, max: 7, port: 3000, wait: 240 }); const url = "http://localhost:3000/data/test.tiff"; @@ -65,13 +65,13 @@ test("Max with Web Mercator Bounding Box and GeoRaster URL", async ({ eq }) => { test("virtual resampling, contained", async ({ eq }) => { const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-one.geojson").then(res => res.json()); - const result = await max(url, geojson) - eq(result, [38]); -}) + const result = await max(url, geojson); + eq(result, [38]); +}); test("virtual resampling, intersecting 4 pixels", async ({ eq }) => { const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-intersect.geojson").then(res => res.json()); - const result = await max(url, geojson) - eq(result, [38]); -}) + const result = await max(url, geojson); + eq(result, [38]); +}); diff --git a/src/mean/index.js b/src/mean/index.js index c484004..f751bd4 100644 --- a/src/mean/index.js +++ b/src/mean/index.js @@ -19,5 +19,5 @@ import stat from "../stat"; */ export default function mean(georaster, geometry, test) { - return stat(georaster, geometry, "mean", test, { vrm: 100 }); + return stat(georaster, geometry, "mean", test, { vrm: "minimal" }); } diff --git a/src/mean/mean.test.js b/src/mean/mean.test.js index ed552aa..ad3659f 100644 --- a/src/mean/mean.test.js +++ b/src/mean/mean.test.js @@ -95,13 +95,28 @@ test("(Modern) Mean from GeoJSON", async ({ eq }) => { test("virtual resampling, contained", async ({ eq }) => { const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-one.geojson").then(res => res.json()); - const result = await mean(url, geojson) - eq(result, [38]); -}) + const result = await mean(url, geojson); + eq(result, [38]); +}); test("virtual resampling, intersecting 4 pixels", async ({ eq }) => { const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-intersect.geojson").then(res => res.json()); - const result = await mean(url, geojson) - eq(result, [33.61065313887127]); -}) \ No newline at end of file + const result = await mean(url, geojson); + eq(result, [38]); +}); + +test("virtual resampling, intersecting 4 pixels (sync)", async ({ eq }) => { + const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; + const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-intersect.geojson").then(res => res.json()); + const georaster = await load(url); + const result = await mean(georaster, geojson); + eq(result, [38]); +}); + +test("virtual resampling with bbox", async ({ eq }) => { + const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; + const geom = [166.06811846012403, -46.42907237702467, 166.23189176587618, -46.40887433963862]; + const result = await mean(url, geom); + eq(result, [38]); +}); diff --git a/src/median/index.js b/src/median/index.js index 1e29289..5323c1f 100644 --- a/src/median/index.js +++ b/src/median/index.js @@ -19,5 +19,5 @@ import stat from "../stat"; */ export default function median(georaster, geometry, test) { - return stat(georaster, geometry, "median", test, { vrm: 100 }); + return stat(georaster, geometry, "median", test, { vrm: "minimal" }); } diff --git a/src/median/median.test.js b/src/median/median.test.js index 48e3625..0e5b637 100644 --- a/src/median/median.test.js +++ b/src/median/median.test.js @@ -82,13 +82,13 @@ test("Get Median from Whole Raster from url", async ({ eq }) => { test("virtual resampling, contained", async ({ eq }) => { const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-one.geojson").then(res => res.json()); - const result = await median(url, geojson) - eq(result, [38]); -}) + const result = await median(url, geojson); + eq(result, [38]); +}); test("virtual resampling, intersecting 4 pixels", async ({ eq }) => { const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-intersect.geojson").then(res => res.json()); - const result = await median(url, geojson) - eq(result, [38]); -}) + const result = await median(url, geojson); + eq(result, [38]); +}); diff --git a/src/min/index.js b/src/min/index.js index 9fd4ecc..7fac1d3 100644 --- a/src/min/index.js +++ b/src/min/index.js @@ -19,5 +19,5 @@ import stat from "../stat"; */ export default function min(georaster, geometry, test) { - return stat(georaster, geometry, "min", test, { vrm: 100 }); + return stat(georaster, geometry, "min", test, { vrm: "minimal" }); } diff --git a/src/min/min.test.js b/src/min/min.test.js index f3e2e73..99e0825 100644 --- a/src/min/min.test.js +++ b/src/min/min.test.js @@ -3,7 +3,7 @@ import { serve } from "srvd"; import load from "../load"; import min from "."; -serve({ debug: true, max: 5, port: 3000 }); +serve({ debug: true, max: 5, port: 3000, wait: 120 }); const url = "http://localhost:3000/data/test.tiff"; const bbox = [80.63, 7.42, 84.21, 10.1]; @@ -65,13 +65,13 @@ test("(Modern) Get Min from whole Raster", async ({ eq }) => { test("virtual resampling, contained", async ({ eq }) => { const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-one.geojson").then(res => res.json()); - const result = await min(url, geojson) - eq(result, [38]); -}) + const result = await min(url, geojson); + eq(result, [38]); +}); test("virtual resampling, intersecting 4 pixels", async ({ eq }) => { const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-intersect.geojson").then(res => res.json()); - const result = await min(url, geojson) - eq(result, [1]); -}) + const result = await min(url, geojson); + eq(result, [38]); +}); diff --git a/src/mode/index.js b/src/mode/index.js index 80b4059..ada8d6b 100644 --- a/src/mode/index.js +++ b/src/mode/index.js @@ -20,5 +20,5 @@ import stat from "../stat"; */ export default function mode(georaster, geometry, test) { - return stat(georaster, geometry, "mode", test, { vrm: 100 }); + return stat(georaster, geometry, "mode", test, { vrm: "minimal" }); } diff --git a/src/mode/mode.test.js b/src/mode/mode.test.js index 9adead1..76dfee6 100644 --- a/src/mode/mode.test.js +++ b/src/mode/mode.test.js @@ -48,13 +48,13 @@ test("(Modern) Mode Polygon", async ({ eq }) => { test("virtual resampling, contained", async ({ eq }) => { const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-one.geojson").then(res => res.json()); - const result = await mode(url, geojson) - eq(result, [38]); -}) + const result = await mode(url, geojson); + eq(result, [38]); +}); test("virtual resampling, intersecting 4 pixels", async ({ eq }) => { const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-intersect.geojson").then(res => res.json()); - const result = await mode(url, geojson) - eq(result, [38]); -}) + const result = await mode(url, geojson); + eq(result, [38]); +}); diff --git a/src/modes/index.js b/src/modes/index.js index 920efe7..b2e52b3 100644 --- a/src/modes/index.js +++ b/src/modes/index.js @@ -20,5 +20,5 @@ import stat from "../stat"; */ export default function modes(georaster, geometry, test) { - return stat(georaster, geometry, "modes", test, { vrm: 100 }); + return stat(georaster, geometry, "modes", test, { vrm: "minimal" }); } diff --git a/src/modes/modes.test.js b/src/modes/modes.test.js index eed02e1..77c0d9d 100644 --- a/src/modes/modes.test.js +++ b/src/modes/modes.test.js @@ -48,13 +48,13 @@ test("(Modern) Modes Polygon", async ({ eq }) => { test("virtual resampling, contained", async ({ eq }) => { const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-one.geojson").then(res => res.json()); - const result = await modes(url, geojson) - eq(result, [38]); -}) + const result = await modes(url, geojson); + eq(result, [[38]]); +}); test("virtual resampling, intersecting 4 pixels", async ({ eq }) => { const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-intersect.geojson").then(res => res.json()); - const result = await modes(url, geojson) - eq(result, [38]); -}) \ No newline at end of file + const result = await modes(url, geojson); + eq(result, [[38]]); +}); diff --git a/src/range/index.js b/src/range/index.js index 37b6e66..99c2a28 100644 --- a/src/range/index.js +++ b/src/range/index.js @@ -19,5 +19,5 @@ import stat from "../stat"; */ export default function range(georaster, geometry, test) { - return stat(georaster, geometry, "range", test, { vrm: 100 }); + return stat(georaster, geometry, "range", test, { vrm: "minimal" }); } diff --git a/src/range/range.test.js b/src/range/range.test.js index 96ca0bf..86456ba 100644 --- a/src/range/range.test.js +++ b/src/range/range.test.js @@ -2,7 +2,9 @@ import test from "flug"; import { serve } from "srvd"; import range from "."; -serve({ debug: true, max: 25, port: 3000 }); +const sleep = ms => new Promise(resolve => setTimeout(() => resolve(), ms)); + +serve({ debug: true, max: 22, port: 3000, wait: 160 }); const url = "http://localhost:3000/data/test.tiff"; const bbox = [80.63, 7.42, 84.21, 10.1]; @@ -40,18 +42,22 @@ test("(Modern) Get Range from Polygon", async ({ eq }) => { test("(Modern) Get Range from whole Raster", async ({ eq }) => { const results = await range(url); eq(results, [8131.2]); + + // weird hack to give time for main thread and/or srvd to clear up tasks before next request is sent + await sleep(1000); }); test("virtual resampling, contained", async ({ eq }) => { + const geojson_res = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-one.geojson"); + const geojson = await geojson_res.json(); const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; - const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-one.geojson").then(res => res.json()); - const result = await range(url, geojson) - eq(result, [0]); -}) + const result = await range(url, geojson); + eq(result, [0]); +}); test("virtual resampling, intersecting 4 pixels", async ({ eq }) => { const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-intersect.geojson").then(res => res.json()); - const result = await range(url, geojson) - eq(result, [37]); -}) + const result = await range(url, geojson); + eq(result, [0]); // range is zero because by default doing minimal (i.e. bare minimum) resampling +}); diff --git a/src/stats/stats.core.js b/src/stats/stats.core.js index f235f3a..6a4233a 100644 --- a/src/stats/stats.core.js +++ b/src/stats/stats.core.js @@ -1,12 +1,16 @@ import calcStats from "calc-stats"; import QuickPromise from "quick-promise"; +import polygon from "bbox-fns/polygon"; +import validate from "bbox-fns/validate"; import get from "../get"; import utils from "../utils"; import wrap from "../wrap-parse"; import { convertBbox, convertMultiPolygon } from "../convert-geometry"; import intersectPolygon from "../intersect-polygon"; -const stats = (georaster, geometry, calcStatsOptions, test, { debug_level = 0, vrm } = {}) => { +const VRM_NO_RESAMPLING = [1, 1]; + +const stats = (georaster, geometry, calcStatsOptions, test, { debug_level = 0, include_meta = false, rescale = false, vrm = VRM_NO_RESAMPLING } = {}) => { try { // shallow clone calcStatsOptions = { ...calcStatsOptions }; @@ -18,20 +22,13 @@ const stats = (georaster, geometry, calcStatsOptions, test, { debug_level = 0, v calcStatsOptions.noData = noDataValue; } - let xvrm; - let yvrm; if (typeof vrm === "number") { - if (vrm !== Math.round(vrm)) { - throw new Error("[geoblaze] divisor must be an integer"); + if (vrm <= 0 || vrm !== Math.round(vrm)) { + throw new Error("[geoblaze] vrm can only be defined as a positive integer"); } - xvrm = vrm; - yvrm = vrm; - } else if (Array.isArray(vrm) && vrm.length === 2 && typeof vrm[0] === "number") { - [xvrm, yvrm] = vrm; + vrm = [vrm, vrm]; } - console.log("vrm:", [xvrm, yvrm]); - if (test) { if (calcStatsOptions && calcStatsOptions.filter) { const original_filter = calcStatsOptions.filter; @@ -44,14 +41,24 @@ const stats = (georaster, geometry, calcStatsOptions, test, { debug_level = 0, v const flat = true; const getStatsByBand = values => values.map(band => calcStats(band, calcStatsOptions)); - if (geometry === null || geometry === undefined) { + const resample = vrm === "minimal" || (Array.isArray(vrm) && (vrm[0] !== 1 || vrm[1] !== 1)); + const geometry_is_nullish = geometry === null || geometry === undefined; + + if (resample === true) { + if (geometry_is_nullish) { + geometry = polygon([georaster.xmin, georaster.ymin, georaster.xmax, georaster.ymax]); + } else if (validate(geometry)) { + geometry = polygon(geometry); + } + } + + if (geometry_is_nullish && resample === false) { if (debug_level >= 2) console.log("[geoblaze] geometry is nullish"); const values = get(georaster, undefined, flat); return QuickPromise.resolve(values).then(getStatsByBand); - } else if (utils.isBbox(geometry)) { + } else if (utils.isBbox(geometry) && resample === false) { if (debug_level >= 2) console.log("[geoblaze] geometry is a rectangle"); geometry = convertBbox(geometry); - // if using multiplier, might need to pad get results or at least not round const values = get(georaster, geometry, flat); return QuickPromise.resolve(values).then(getStatsByBand); } else if (utils.isPolygonal(geometry)) { @@ -77,10 +84,85 @@ const stats = (georaster, geometry, calcStatsOptions, test, { debug_level = 0, v { debug_level, vrm } ); - return QuickPromise.resolve(done).then(() => { + return QuickPromise.resolve(done).then(({ vrm }) => { + // check if the user wants the number of valid pixels returned + const want_valid = calcStatsOptions.stats ? calcStatsOptions.stats.includes("valid") : calcStatsOptions.calcValid === true; + + const use_virtual_resampling = vrm[0] !== 1 && vrm[1] !== 1; + const bands = values.filter(band => band.length !== 0); - if (bands.length > 0) return bands.map(band => calcStats(band, calcStatsOptions)); - else throw "No Values were found in the given geometry"; + + if (bands.length > 0) { + // if we need to know the number of valid pixels for intermediate calculations, + // but the user didn't ask for it, calculate it anyway + // and then remove the valid count from the returned results + if (use_virtual_resampling) { + if (calcStatsOptions.stats) { + if (calcStatsOptions.stats.includes("product") && calcStatsOptions.stats.includes("valid") === false) { + calcStatsOptions.stats = [...calcStatsOptions.stats, "valid"]; + if (debug_level >= 2) console.log('[geoblaze] added "valid" to stats'); + } + } else { + if (calcStatsOptions.calcProduct === true && calcStatsOptions.calcValid === false) { + calcStatsOptions.calcValid = true; + if (debug_level >= 2) console.log('[geoblaze] set "calcValid" to true'); + } + } + } + + const stats = bands.map(band => calcStats(band, calcStatsOptions)); + if (debug_level >= 2) console.log("[geoblaze] stats (before rescaling):", stats); + + // only rescaling results if virtual resampling is on + if (use_virtual_resampling && rescale) { + if (debug_level >= 2) console.log("[geoblaze] rescaling results based on relative size of virtual pixels"); + + // if vrm is [2, 4] then area_multiplier will be 8, + // meaning there are 8 virtual pixels for every actual pixel + const area_multiplier = vrm[0] * vrm[1]; + if (debug_level >= 2) console.log("[geoblaze] area_multiplier:", area_multiplier); + + stats.forEach(band => { + const { valid } = band; + if (typeof band.count === "number") band.count /= area_multiplier; + if (typeof band.invalid === "number") band.invalid /= area_multiplier; + if (typeof band.sum === "number") band.sum /= area_multiplier; + if (typeof band.valid === "number") band.valid /= area_multiplier; + + if (band.histogram) { + for (let key in band.histogram) { + // this will lead to fractions of pixels + // for example, a pixel could appear 3.75 times if vrm is [2, 2] + band.histogram[key].ct /= area_multiplier; + } + } + + if (typeof band.product === "number") { + band.product /= Math.pow(area_multiplier, valid); + } + }); + } + + // if the user asked not to have the valid stat, + // exclude if from the results + if (want_valid === false) { + stats.forEach(band => delete band.valid); + } + + if (include_meta) { + stats.forEach(band => { + band._meta = { + vph: georaster.pixelHeight / vrm[1], + vpw: georaster.pixelWidth / vrm[0], + vrm: [vrm[0], vrm[1]] + }; + }); + } + + return stats; + } else { + throw "No Values were found in the given geometry"; + } }); } else { throw "Geometry Type is Not Supported"; diff --git a/src/stats/stats.test.js b/src/stats/stats.test.js index 3025307..09c2a88 100644 --- a/src/stats/stats.test.js +++ b/src/stats/stats.test.js @@ -10,7 +10,7 @@ import load from "../load"; import parse from "../parse"; import stats from "./stats.module"; -serve({ debug: true, max: 30, port: 3000, wait: 240 }); +serve({ debug: true, max: 35, port: 3000, wait: 240 }); const url = "http://localhost:3000/data/test.tiff"; @@ -244,3 +244,66 @@ test("multipolygon vs 2 polygons", async ({ eq }) => { const results2 = await stats(georaster, poly2, { debug_level: 5, stats: _stats }); eq(results2, [{ count: 576, valid: 4, invalid: 572, min: 0, max: 0, sum: 0 }]); }); + +test("virtual resampling, contained", async ({ eq }) => { + const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; + const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-one.geojson").then(res => res.json()); + const result = await stats(url, geojson, undefined, undefined, { debug_level: 10, rescale: true, vrm: "minimal" }); + eq(result, [ + { + count: 0.007936507936507936, + invalid: 0, + median: 38, + min: 38, + max: 38, + product: 0.022738725119677502, + sum: 0.30158730158730157, + range: 0, + mean: 38, + variance: 0, + std: 0, + histogram: { 38: { n: 38, ct: 0.007936507936507936 } }, + modes: [38], + mode: 38, + uniques: [38] + } + ]); +}); + +test("virtual resampling, intersecting 4 pixels", async ({ eq }) => { + const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; + const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-intersect.geojson").then(res => res.json()); + const result = await stats(url, geojson, null, null, { include_meta: false, rescale: true, vrm: [10, 10] }); + eq(result, [ + { + count: 0.18, + invalid: 0, + median: 38, + min: 1, + max: 38, + product: 1.5122637183654097e-10, + sum: 6.17, + range: 37, + mean: 34.27777777777778, + variance: 112.20061728395062, + std: 10.592479279373201, + histogram: { + 1: { + n: 1, + ct: 0.01 + }, + 8: { + n: 8, + ct: 0.01 + }, + 38: { + n: 38, + ct: 0.16 + } + }, + modes: [38], + mode: 38, + uniques: [1, 8, 38] + } + ]); +}); diff --git a/src/sum/index.js b/src/sum/index.js index 7eebc35..75659af 100644 --- a/src/sum/index.js +++ b/src/sum/index.js @@ -1,3 +1,4 @@ +import QuickPromise from "quick-promise"; import stat from "../stat"; /** @@ -17,6 +18,6 @@ import stat from "../stat"; * // results is [red, green, blue, nir] * [217461, 21375, 57312, 457125] */ -export default function sum(georaster, geometry, test, { vrm } = {}) { - return stat(georaster, geometry, "sum", test, { vrm }); +export default function sum(georaster, geometry, test) { + return QuickPromise.resolve(stat(georaster, geometry, "sum", test, { rescale: true, vrm: "minimal" })); } diff --git a/src/sum/sum.test.js b/src/sum/sum.test.js index 24db67e..da5639d 100644 --- a/src/sum/sum.test.js +++ b/src/sum/sum.test.js @@ -312,6 +312,6 @@ test("Virtual Resampling", async ({ eq }) => { await fetchJson(`http://localhost:${port}/data/virtual-resampling/virtual-resampling-one.geojson`) ]; const [georaster, geojson] = values; - const results = await sum(georaster, geojson, undefined, { vrm: 10 }); - eq(results, []); + const results = await sum(georaster, geojson, undefined); + eq(results, [0.30158730158730157]); }); From 70c3297215c60089f0d757d0359b23af86f5367c Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Sat, 11 May 2024 09:57:27 -0400 Subject: [PATCH 05/23] fixed lint issue --- src/stats/stats.core.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stats/stats.core.js b/src/stats/stats.core.js index 6a4233a..f641f25 100644 --- a/src/stats/stats.core.js +++ b/src/stats/stats.core.js @@ -130,7 +130,7 @@ const stats = (georaster, geometry, calcStatsOptions, test, { debug_level = 0, i if (typeof band.valid === "number") band.valid /= area_multiplier; if (band.histogram) { - for (let key in band.histogram) { + for (const key in band.histogram) { // this will lead to fractions of pixels // for example, a pixel could appear 3.75 times if vrm is [2, 2] band.histogram[key].ct /= area_multiplier; From bcfa92f2b405c91758e4dcac634453c94c87881b Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Sat, 11 May 2024 10:14:57 -0400 Subject: [PATCH 06/23] increased served requests in intersect-polygon.test.js --- src/intersect-polygon/intersect-polygon.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/intersect-polygon/intersect-polygon.test.js b/src/intersect-polygon/intersect-polygon.test.js index d7ee1d4..0c3fa7a 100644 --- a/src/intersect-polygon/intersect-polygon.test.js +++ b/src/intersect-polygon/intersect-polygon.test.js @@ -39,7 +39,7 @@ async function fetch_json(url) { } } -serve({ debug: true, max: 25, port: 3000, wait: 60 }); +serve({ debug: true, max: 30, port: 3000, wait: 60 }); const urlToGeojson = "http://localhost:3000/data/gadm/geojsons/Akrotiri and Dhekelia.geojson"; From f601b6b56ba4d854decccec6f818ebb1930f2a47 Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Sat, 11 May 2024 12:21:58 -0400 Subject: [PATCH 07/23] updated flug and added 1 sec gap time between tests --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 28393ca..6bdd133 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "format": "npx prettier --arrow-parens=avoid --print-width=160 --trailing-comma=none --write lite/*.js src/*.js src/*/*.js ", "lint": "eslint lite && eslint src", "serve": "npx srvd --debug --port=3000 --wait=120", - "test": "set -e; for f in src/*/*test*.js; do echo \"\nrunning $f\" && sleep 5 && node -r esm $f; done", + "test": "set -e; for f in src/*/*test*.js; do echo \"\nrunning $f\" && sleep 5 && TEST_GAP_TIME=1 node -r esm $f; done", "test:intersect-polygon": "node -r esm ./src/intersect-polygon/intersect-polygon.test.js", "test:max": "node -r esm ./src/max/max.test.js", "test:mean": "node -r esm ./src/mean/mean.test.js", @@ -124,7 +124,7 @@ "eslint-plugin-standard": "^3.1.0", "esm": "3.2.25", "find-and-read": "^1.2.0", - "flug": "^2.6.0", + "flug": "^2.8.0", "jsdoc": "^3.6.7", "memory-fs": "^0.4.1", "null-loader": "^4.0.1", From 9f2d7f15b3e9c3f6395cf5244d3141c8bd3ded13 Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Sun, 12 May 2024 08:47:41 -0400 Subject: [PATCH 08/23] updated flug --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6bdd133..beb90f1 100644 --- a/package.json +++ b/package.json @@ -124,7 +124,7 @@ "eslint-plugin-standard": "^3.1.0", "esm": "3.2.25", "find-and-read": "^1.2.0", - "flug": "^2.8.0", + "flug": "^2.8.1", "jsdoc": "^3.6.7", "memory-fs": "^0.4.1", "null-loader": "^4.0.1", From 4017f1ff73263d58d1cd86aa1d23a6f323c80040 Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Sun, 12 May 2024 08:57:54 -0400 Subject: [PATCH 09/23] added virtual resampling test files --- .../virtual-resampling-intersect.geojson | 8 ++++++++ data/virtual-resampling/virtual-resampling-one.geojson | 8 ++++++++ 2 files changed, 16 insertions(+) create mode 100644 data/virtual-resampling/virtual-resampling-intersect.geojson create mode 100644 data/virtual-resampling/virtual-resampling-one.geojson diff --git a/data/virtual-resampling/virtual-resampling-intersect.geojson b/data/virtual-resampling/virtual-resampling-intersect.geojson new file mode 100644 index 0000000..3662829 --- /dev/null +++ b/data/virtual-resampling/virtual-resampling-intersect.geojson @@ -0,0 +1,8 @@ +{ +"type": "FeatureCollection", +"name": "virtual-resampling-intersect", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 165.939001402474787, -46.486236933721152 ], [ 166.303394074387001, -46.121247183868078 ], [ 166.581806452926685, -46.392534642381364 ], [ 165.996321598056511, -46.660679777789575 ], [ 165.939001402474787, -46.486236933721152 ] ] ] } } +] +} diff --git a/data/virtual-resampling/virtual-resampling-one.geojson b/data/virtual-resampling/virtual-resampling-one.geojson new file mode 100644 index 0000000..c1dd305 --- /dev/null +++ b/data/virtual-resampling/virtual-resampling-one.geojson @@ -0,0 +1,8 @@ +{ +"type": "FeatureCollection", +"name": "virtual-resampling-one", +"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, +"features": [ +{ "type": "Feature", "properties": { }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 166.128374676391331, -46.408874339638622 ], [ 166.231891765876185, -46.50319516961985 ], [ 166.051123117074297, -46.506566757293193 ], [ 166.068118460124026, -46.429072377024667 ], [ 166.128374676391331, -46.408874339638622 ] ] ] } } +] +} From 56c9e1633774b42195a6e06cbdebaae9ecb4d769 Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Sun, 12 May 2024 09:19:26 -0400 Subject: [PATCH 10/23] serve more files in median test --- src/median/median.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/median/median.test.js b/src/median/median.test.js index 0e5b637..e415387 100644 --- a/src/median/median.test.js +++ b/src/median/median.test.js @@ -4,7 +4,7 @@ import { serve } from "srvd"; import load from "../load"; import median from "."; -serve({ debug: true, max: 1, port: 3000 }); +serve({ debug: true, max: 6, port: 3000 }); const url = "http://localhost:3000/data/test.tiff"; @@ -79,7 +79,7 @@ test("Get Median from Whole Raster from url", async ({ eq }) => { eq(value, 0); }); -test("virtual resampling, contained", async ({ eq }) => { +test("Get Median with virtual resampling, contained", async ({ eq }) => { const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-one.geojson").then(res => res.json()); const result = await median(url, geojson); From 8b5035f9f133997a1ef47cb6ad70ea7d9bd27355 Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Sun, 12 May 2024 10:33:14 -0400 Subject: [PATCH 11/23] updated stats to return valid by default --- src/stats/stats.core.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stats/stats.core.js b/src/stats/stats.core.js index f641f25..d96b11f 100644 --- a/src/stats/stats.core.js +++ b/src/stats/stats.core.js @@ -85,8 +85,8 @@ const stats = (georaster, geometry, calcStatsOptions, test, { debug_level = 0, i ); return QuickPromise.resolve(done).then(({ vrm }) => { - // check if the user wants the number of valid pixels returned - const want_valid = calcStatsOptions.stats ? calcStatsOptions.stats.includes("valid") : calcStatsOptions.calcValid === true; + // check if the user doesn't want the number of valid pixels returned + const want_valid = calcStatsOptions.stats ? calcStatsOptions.stats.includes("valid") : calcStatsOptions.calcValid !== false; const use_virtual_resampling = vrm[0] !== 1 && vrm[1] !== 1; From 49371f0c21c742feb6da3f4646e9380afcd1b2bc Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Sun, 12 May 2024 11:56:38 -0400 Subject: [PATCH 12/23] removed old code climate links --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 24276ab..6706a15 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,5 @@ -[![Maintainability](https://api.codeclimate.com/v1/badges/a99a88d28ad37a79dbf6/maintainability)](https://codeclimate.com/github/codeclimate/codeclimate/maintainability) -[![Test Coverage](https://api.codeclimate.com/v1/badges/a99a88d28ad37a79dbf6/test_coverage)](https://codeclimate.com/github/codeclimate/codeclimate/test_coverage) - # GeoBlaze ***A blazing fast javascript raster processing engine*** From dfb76b6746c481174da53361d2ebeb3246c0ffaa Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Wed, 15 May 2024 22:31:47 -0400 Subject: [PATCH 13/23] updated flug --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index beb90f1..82b59ac 100644 --- a/package.json +++ b/package.json @@ -124,7 +124,7 @@ "eslint-plugin-standard": "^3.1.0", "esm": "3.2.25", "find-and-read": "^1.2.0", - "flug": "^2.8.1", + "flug": "^2.8.2", "jsdoc": "^3.6.7", "memory-fs": "^0.4.1", "null-loader": "^4.0.1", From 25f5fdb7eed267960895bcd5fb1b66ead0310cd3 Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Thu, 16 May 2024 07:57:18 -0400 Subject: [PATCH 14/23] added scaled valid values to stats test --- src/stats/stats.test.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/stats/stats.test.js b/src/stats/stats.test.js index 09c2a88..41f7f5d 100644 --- a/src/stats/stats.test.js +++ b/src/stats/stats.test.js @@ -260,6 +260,7 @@ test("virtual resampling, contained", async ({ eq }) => { sum: 0.30158730158730157, range: 0, mean: 38, + valid: 0.007936507936507936, variance: 0, std: 0, histogram: { 38: { n: 38, ct: 0.007936507936507936 } }, @@ -285,6 +286,7 @@ test("virtual resampling, intersecting 4 pixels", async ({ eq }) => { sum: 6.17, range: 37, mean: 34.27777777777778, + valid: 0.18, variance: 112.20061728395062, std: 10.592479279373201, histogram: { From c8475fb0e6a0ea10f56655df64310c80c63707d7 Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Thu, 16 May 2024 22:08:18 -0400 Subject: [PATCH 15/23] removed frequency from stats tests --- src/stats/stats.test.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/stats/stats.test.js b/src/stats/stats.test.js index 41f7f5d..7c690d8 100644 --- a/src/stats/stats.test.js +++ b/src/stats/stats.test.js @@ -79,8 +79,9 @@ test("(Sync) Stats without Geometry", async ({ eq }) => { const georaster = await load(url); const results = stats(georaster, undefined); results.forEach(band => { - delete band.uniques; + delete band.frequency; delete band.histogram; + delete band.uniques; }); eq(results, EXPECTED_RASTER_STATS); }); @@ -88,6 +89,7 @@ test("(Sync) Stats without Geometry", async ({ eq }) => { test("(Async) Stats without Geometry", async ({ eq }) => { const results = await stats(url, undefined); results.forEach(band => { + delete band.frequency; delete band.histogram; delete band.uniques; }); @@ -98,6 +100,7 @@ test("(Sync) Stats with Bounding Box", async ({ eq }) => { const georaster = await load(url); const results = stats(georaster, bbox); results.forEach(band => { + delete band.frequency; delete band.histogram; delete band.uniques; }); @@ -107,6 +110,7 @@ test("(Sync) Stats with Bounding Box", async ({ eq }) => { test("(Async) Stats with Bounding Box", async ({ eq }) => { const results = await stats(url, bbox); results.forEach(band => { + delete band.frequency; delete band.histogram; delete band.uniques; }); @@ -117,6 +121,7 @@ test("(Sync) Stats with Polygon", async ({ eq }) => { const georaster = await load(url); const results = stats(georaster, polygon); results.forEach(band => { + delete band.frequency; delete band.histogram; delete band.uniques; }); @@ -126,6 +131,7 @@ test("(Sync) Stats with Polygon", async ({ eq }) => { test("(Async) Stats with Polygon", async ({ eq }) => { const results = await stats(url, polygon); results.forEach(band => { + delete band.frequency; delete band.histogram; delete band.uniques; }); From 846cb74208b8c79636927490d0ef4decf10ef317 Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Thu, 6 Jun 2024 21:32:44 -0400 Subject: [PATCH 16/23] added frequency to test expectation --- src/stats/stats.test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/stats/stats.test.js b/src/stats/stats.test.js index 7c690d8..2a39345 100644 --- a/src/stats/stats.test.js +++ b/src/stats/stats.test.js @@ -258,6 +258,7 @@ test("virtual resampling, contained", async ({ eq }) => { eq(result, [ { count: 0.007936507936507936, + frequency: { 38: { n: 38, freq: 1 } }, invalid: 0, median: 38, min: 38, From 2101fa426a0e3c455018bfa22789097a0e0ef01d Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Fri, 7 Jun 2024 21:49:24 -0400 Subject: [PATCH 17/23] removed debug logging --- src/stats/stats.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stats/stats.test.js b/src/stats/stats.test.js index 2a39345..2dd8c29 100644 --- a/src/stats/stats.test.js +++ b/src/stats/stats.test.js @@ -254,7 +254,7 @@ test("multipolygon vs 2 polygons", async ({ eq }) => { test("virtual resampling, contained", async ({ eq }) => { const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-one.geojson").then(res => res.json()); - const result = await stats(url, geojson, undefined, undefined, { debug_level: 10, rescale: true, vrm: "minimal" }); + const result = await stats(url, geojson, undefined, undefined, { debug_level: 0, rescale: true, vrm: "minimal" }); eq(result, [ { count: 0.007936507936507936, From 63b1e905d795fa1e0d3d5442c2492437e5a7d6a2 Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Fri, 7 Jun 2024 22:19:08 -0400 Subject: [PATCH 18/23] try to fix ci testing issues --- src/stats/stats.test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/stats/stats.test.js b/src/stats/stats.test.js index 2dd8c29..07313e7 100644 --- a/src/stats/stats.test.js +++ b/src/stats/stats.test.js @@ -279,6 +279,7 @@ test("virtual resampling, contained", async ({ eq }) => { }); test("virtual resampling, intersecting 4 pixels", async ({ eq }) => { + await new Promise(resolve => setTimeout(resolve, 5*1000)); // avoid failed fetching const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-intersect.geojson").then(res => res.json()); const result = await stats(url, geojson, null, null, { include_meta: false, rescale: true, vrm: [10, 10] }); From f53c595094c5c7d836650db505ff02bb81e11bca Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Sat, 8 Jun 2024 09:53:38 -0400 Subject: [PATCH 19/23] changed debug level --- src/stats/stats.test.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/stats/stats.test.js b/src/stats/stats.test.js index 07313e7..f3cc74b 100644 --- a/src/stats/stats.test.js +++ b/src/stats/stats.test.js @@ -240,14 +240,14 @@ test("multipolygon vs 2 polygons", async ({ eq }) => { const _stats = ["count", "invalid", "min", "max", "sum", "valid"]; const expected = [{ count: 1152, valid: 4, invalid: 1148, min: 0, max: 0, sum: 0 }]; - eq(await stats(georaster, geojson, { stats: _stats }, undefined, { debug_level: 5 }), expected); + eq(await stats(georaster, geojson, { stats: _stats }, undefined, { debug_level: 0 }), expected); eq(await stats(georaster, geojson.features[0], { stats: _stats }), expected); const [poly1, poly2] = geojson.features[0].geometry.coordinates; - const results1 = await stats(georaster, poly1, { debug_level: 5, stats: _stats }); + const results1 = await stats(georaster, poly1, { debug_level: 0, stats: _stats }); eq(results1, [{ count: 576, valid: 0, invalid: 576, sum: 0, min: undefined, max: undefined }]); - const results2 = await stats(georaster, poly2, { debug_level: 5, stats: _stats }); + const results2 = await stats(georaster, poly2, { debug_level: 0, stats: _stats }); eq(results2, [{ count: 576, valid: 4, invalid: 572, min: 0, max: 0, sum: 0 }]); }); From 4faff440d7edac483489b2cacf1f81aa10824ede Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Sat, 8 Jun 2024 10:04:37 -0400 Subject: [PATCH 20/23] trying to fix weird ci issue with fetching file by using readFileSync instead --- src/stats/stats.test.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/stats/stats.test.js b/src/stats/stats.test.js index f3cc74b..2eca047 100644 --- a/src/stats/stats.test.js +++ b/src/stats/stats.test.js @@ -279,9 +279,8 @@ test("virtual resampling, contained", async ({ eq }) => { }); test("virtual resampling, intersecting 4 pixels", async ({ eq }) => { - await new Promise(resolve => setTimeout(resolve, 5*1000)); // avoid failed fetching const url = "http://localhost:3000/data/geotiff-test-data/nz_habitat_anticross_4326_1deg.tif"; - const geojson = await fetch("http://localhost:3000/data/virtual-resampling/virtual-resampling-intersect.geojson").then(res => res.json()); + const geojson = JSON.parse(readFileSync("./data/virtual-resampling/virtual-resampling-intersect.geojson", "utf-8")); const result = await stats(url, geojson, null, null, { include_meta: false, rescale: true, vrm: [10, 10] }); eq(result, [ { From ae555193f7134ce7328851eb8dd1104a541e3d3d Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Sat, 8 Jun 2024 10:23:47 -0400 Subject: [PATCH 21/23] added frequency to expected test result --- src/stats/stats.test.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/stats/stats.test.js b/src/stats/stats.test.js index 2eca047..585e651 100644 --- a/src/stats/stats.test.js +++ b/src/stats/stats.test.js @@ -296,6 +296,20 @@ test("virtual resampling, intersecting 4 pixels", async ({ eq }) => { valid: 0.18, variance: 112.20061728395062, std: 10.592479279373201, + frequency: { + 1: { + freq: 0.05555555555555555, + n: 1 + }, + 8: { + freq: 0.05555555555555555, + n: 8 + }, + 38: { + freq: 0.8888888888888888, + n: 38 + } + }, histogram: { 1: { n: 1, From bf45190f55e44e6c287f80a3f5a64821a4de9cbf Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Sat, 8 Jun 2024 11:41:25 -0400 Subject: [PATCH 22/23] add support for simple bbox obj resampling --- src/stats/stats.core.js | 3 +++ src/sum/index.js | 2 +- src/sum/sum.test.js | 3 ++- src/utils/utils.module.js | 12 ++++++++++++ 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/stats/stats.core.js b/src/stats/stats.core.js index d96b11f..f5343de 100644 --- a/src/stats/stats.core.js +++ b/src/stats/stats.core.js @@ -49,6 +49,9 @@ const stats = (georaster, geometry, calcStatsOptions, test, { debug_level = 0, i geometry = polygon([georaster.xmin, georaster.ymin, georaster.xmax, georaster.ymax]); } else if (validate(geometry)) { geometry = polygon(geometry); + } else if (utils.isBboxObj(geometry)) { + // convert { xmin: 20, xmax: 32, ymin: -3, ymax: 0 } to geojson polygon + geometry = polygon([geometry.xmin, geometry.ymin, geometry.xmax, geometry.ymax]); } } diff --git a/src/sum/index.js b/src/sum/index.js index 75659af..0ced398 100644 --- a/src/sum/index.js +++ b/src/sum/index.js @@ -19,5 +19,5 @@ import stat from "../stat"; * [217461, 21375, 57312, 457125] */ export default function sum(georaster, geometry, test) { - return QuickPromise.resolve(stat(georaster, geometry, "sum", test, { rescale: true, vrm: "minimal" })); + return stat(georaster, geometry, "sum", test, { rescale: true, vrm: "minimal" }); } diff --git a/src/sum/sum.test.js b/src/sum/sum.test.js index da5639d..7b17461 100644 --- a/src/sum/sum.test.js +++ b/src/sum/sum.test.js @@ -13,7 +13,7 @@ import sum from "."; const { fetchJson, fetchJsons } = utils; -const { port } = serve({ debug: true, max: 100, port: 8888, wait: 120 }); +const { port } = serve({ debug: true, max: 31, port: 8888, wait: 120 }); const urlRwanda = `http://localhost:${port}/data/RWA_MNH_ANC.tif`; const bboxRwanda = require("../../data/RwandaBufferedBoundingBox.json"); @@ -59,6 +59,7 @@ test("(Legacy) Get Sum from Veneto Geonode", async ({ eq }) => { ]; const [georaster, geojson] = values; const results = sum(georaster, geojson); + eq(Array.isArray(results), true); const actualValue = Number(results[0].toFixed(2)); const expectedValue = 24_943.31; // rasterstats says 24,963.465454101562 eq(actualValue, expectedValue); diff --git a/src/utils/utils.module.js b/src/utils/utils.module.js index 91255e6..5e5034b 100644 --- a/src/utils/utils.module.js +++ b/src/utils/utils.module.js @@ -122,6 +122,18 @@ const utils = { return Array.isArray(parsed) ? parsed[0] : parsed; }, + // checks if bbox in form { xmin: 20, xmax: 32, ymin: -3, ymax: 0 } + isBboxObj(geometry) { + return ( + typeof geometry === "object" && + Array.isArray(geometry) === false && + typeof geometry.xmin === "number" && + typeof geometry.xmax === "number" && + typeof geometry.ymin === "number" && + typeof geometry.ymax === "number" + ); + }, + isPolygonal(geometry) { const polys = mpoly.get(geometry); From 9645c6930d793ae3c4a1ec7b03eec64f99a66607 Mon Sep 17 00:00:00 2001 From: DanielJDufour Date: Sat, 8 Jun 2024 11:46:20 -0400 Subject: [PATCH 23/23] fix linting issue by removing unused import in src/sum/index.js --- src/sum/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sum/index.js b/src/sum/index.js index 0ced398..1751194 100644 --- a/src/sum/index.js +++ b/src/sum/index.js @@ -1,4 +1,3 @@ -import QuickPromise from "quick-promise"; import stat from "../stat"; /**