From a1fa9dee336ca8fcfa58132a93217c060203262c Mon Sep 17 00:00:00 2001 From: Vojta Tomas Date: Wed, 8 Jan 2025 20:15:45 +0000 Subject: [PATCH 1/3] fix: adding a mechanism to check for presence of m-values during parsing --- shpts/geometry/base.ts | 16 +++++++++++++--- shpts/geometry/multipatch.ts | 12 ++++++++---- shpts/geometry/multipoint.ts | 15 +++++++++++---- shpts/geometry/null.ts | 2 +- shpts/geometry/point.ts | 17 +++++++---------- shpts/geometry/polygon.ts | 11 +++++++---- shpts/geometry/polyline.ts | 11 +++++++---- shpts/reader/shpReader.ts | 22 ++++++++++++++++++++-- shpts/shpts.ts | 3 ++- shpts/types/data.ts | 3 +++ shpts/utils/geometry.ts | 2 +- test/polygon.test.ts | 20 +++++++++++++++++++- testdata/polygonZ.cpg | 1 + testdata/polygonZ.dbf | Bin 0 -> 78 bytes testdata/polygonZ.prj | 1 + testdata/polygonZ.shp | Bin 0 -> 588 bytes testdata/polygonZ.shx | Bin 0 -> 108 bytes 17 files changed, 101 insertions(+), 35 deletions(-) create mode 100644 testdata/polygonZ.cpg create mode 100644 testdata/polygonZ.dbf create mode 100644 testdata/polygonZ.prj create mode 100644 testdata/polygonZ.shp create mode 100644 testdata/polygonZ.shx diff --git a/shpts/geometry/base.ts b/shpts/geometry/base.ts index 95270f7..6e800d7 100644 --- a/shpts/geometry/base.ts +++ b/shpts/geometry/base.ts @@ -1,20 +1,24 @@ import { Coord, CoordType } from '@shpts/types/coordinate'; import { GeoJsonGeom } from '@shpts/types/geojson'; -import { BoundingBox } from '@shpts/types/data'; +import { BoundingBox, GeomHeader } from '@shpts/types/data'; import { MemoryStream } from '@shpts/utils/stream'; export abstract class BaseRecord { - constructor(readonly coordType: CoordType) {} + constructor(readonly coordType: CoordType, private hasMValuesPresent: boolean) {} abstract toGeoJson(): GeoJsonGeom; get hasZ(): boolean { return this.coordType === CoordType.XYZM; } - get hasM(): boolean { + get hasOptionalM(): boolean { return this.coordType === CoordType.XYM || this.coordType === CoordType.XYZM; } + get hasM(): boolean { + return this.hasMValuesPresent; + } + get coordLength(): number { if (this.coordType === CoordType.XY) return 2; if (this.coordType === CoordType.XYM) return 3; @@ -35,6 +39,12 @@ export abstract class BaseRecord { yMax: yMax, }; } + + static recordReadingFinalized(shpStream: MemoryStream, header: GeomHeader) { + //per spec, each record is (4 + header.length) * 2 bytes long + //if the stream is at the end of the record, then we are good + return shpStream.tell === header.offset + (4 + header.length) * 2; + } } export abstract class BaseRingedRecord extends BaseRecord { diff --git a/shpts/geometry/multipatch.ts b/shpts/geometry/multipatch.ts index 40c2c4d..9adf8c3 100644 --- a/shpts/geometry/multipatch.ts +++ b/shpts/geometry/multipatch.ts @@ -12,8 +12,8 @@ import { assemblePolygonsWithHoles } from '@shpts/utils/orientation'; import { GeomUtil } from '@shpts/utils/geometry'; export class MultiPatchRecord extends BaseRingedRecord { - constructor(public coords: MultiPatchCoord, coordType: CoordType) { - super(coordType); + constructor(public coords: MultiPatchCoord, coordType: CoordType, hasMValuesPresent: boolean) { + super(coordType, hasMValuesPresent); } get type() { @@ -23,7 +23,7 @@ export class MultiPatchRecord extends BaseRingedRecord { static fromPresetReader(reader: ShapeReader, header: GeomHeader) { const hasZ = reader.hasZ; - const hasM = reader.hasM; + const hasOptionalM = reader.hasOptionalM; const shpStream = reader.shpStream; let z, m; @@ -34,12 +34,16 @@ export class MultiPatchRecord extends BaseRingedRecord { const partTypes = shpStream.readInt32Array(numParts, true); const xy = shpStream.readDoubleArray(numPoints * 2, true); if (hasZ) z = MultiPatchRecord.getZValues(shpStream, numPoints); + + const hasM = !this.recordReadingFinalized(shpStream, header) && hasOptionalM; if (hasM) m = MultiPatchRecord.getMValues(shpStream, numPoints); + const coords = MultiPatchRecord.getCoords(parts, xy, z, m); const assembledCoords = MultiPatchRecord.assemblePolygonsWithHoles(coords, partTypes); return new MultiPatchRecord( assembledCoords as MultiPatchCoord, - GeomUtil.coordType(header.type) + GeomUtil.coordType(header.type), + hasM ); } diff --git a/shpts/geometry/multipoint.ts b/shpts/geometry/multipoint.ts index 52e6c1d..57de0f0 100644 --- a/shpts/geometry/multipoint.ts +++ b/shpts/geometry/multipoint.ts @@ -7,8 +7,8 @@ import { GeomHeader } from '@shpts/types/data'; import { MemoryStream } from '@shpts/utils/stream'; export class MultiPointRecord extends BaseRecord { - constructor(public coords: MultiPointCoord, coordType: CoordType) { - super(coordType); + constructor(public coords: MultiPointCoord, coordType: CoordType, hasMValuesPresent: boolean) { + super(coordType, hasMValuesPresent); } get type() { @@ -17,7 +17,7 @@ export class MultiPointRecord extends BaseRecord { static fromPresetReader(reader: ShapeReader, header: GeomHeader) { const hasZ = reader.hasZ; - const hasM = reader.hasM; + const hasOptionalM = reader.hasOptionalM; const shpStream = reader.shpStream; let z, m; @@ -25,9 +25,16 @@ export class MultiPointRecord extends BaseRecord { const numPoints = shpStream.readInt32(true); const xy = shpStream.readDoubleArray(numPoints * 2, true); if (hasZ) z = MultiPointRecord.getZValues(shpStream, numPoints); + + const hasM = !this.recordReadingFinalized(shpStream, header) && hasOptionalM; if (hasM) m = MultiPointRecord.getMValues(shpStream, numPoints); + const coords = MultiPointRecord.getCoords(numPoints, xy, z, m); - return new MultiPointRecord(coords as MultiPointCoord, GeomUtil.coordType(header.type)); + return new MultiPointRecord( + coords as MultiPointCoord, + GeomUtil.coordType(header.type), + hasM + ); } private static getZValues(shpStream: MemoryStream, numPoints: number) { diff --git a/shpts/geometry/null.ts b/shpts/geometry/null.ts index c3cfe2b..e8ab69a 100644 --- a/shpts/geometry/null.ts +++ b/shpts/geometry/null.ts @@ -4,7 +4,7 @@ import { CoordType } from '@shpts/types/coordinate'; export class ShpNullGeom extends BaseRecord { constructor() { - super(CoordType.NULL); + super(CoordType.NULL, false); } get type() { diff --git a/shpts/geometry/point.ts b/shpts/geometry/point.ts index e0f8437..355b988 100644 --- a/shpts/geometry/point.ts +++ b/shpts/geometry/point.ts @@ -6,8 +6,8 @@ import { GeomHeader } from '@shpts/types/data'; import { GeomUtil } from '@shpts/utils/geometry'; export class PointRecord extends BaseRecord { - constructor(public coords: PointCoord, coordType: CoordType) { - super(coordType); + constructor(public coords: PointCoord, coordType: CoordType, hasMValuesPresent: boolean) { + super(coordType, hasMValuesPresent); } get type() { @@ -16,22 +16,19 @@ export class PointRecord extends BaseRecord { static fromPresetReader(reader: ShapeReader, header: GeomHeader) { const hasZ = reader.hasZ; - const hasM = reader.hasM; + const hasOptionalM = reader.hasOptionalM; const shpStream = reader.shpStream; const coord = []; coord.push(shpStream.readDouble(true)); //x coord.push(shpStream.readDouble(true)); //y + if (hasZ) coord.push(shpStream.readDouble(true)); //z - if (hasM) { - if (hasZ) { - coord.push(shpStream.readDouble(true)); //z - } - coord.push(shpStream.readDouble(true)); //m - } + const hasM = !this.recordReadingFinalized(shpStream, header) && hasOptionalM; + if (hasM) coord.push(shpStream.readDouble(true)); //m - return new PointRecord(coord as PointCoord, GeomUtil.coordType(header.type)); + return new PointRecord(coord as PointCoord, GeomUtil.coordType(header.type), hasM); } toGeoJson() { diff --git a/shpts/geometry/polygon.ts b/shpts/geometry/polygon.ts index 1a8f77c..0c30c64 100644 --- a/shpts/geometry/polygon.ts +++ b/shpts/geometry/polygon.ts @@ -12,8 +12,8 @@ import { GeomHeader } from '@shpts/types/data'; import { assemblePolygonsWithHoles } from '@shpts/utils/orientation'; export class PolygonRecord extends BaseRingedRecord { - constructor(public coords: PolygonCoord, coordType: CoordType) { - super(coordType); + constructor(public coords: PolygonCoord, coordType: CoordType, hasMValuesPresent: boolean) { + super(coordType, hasMValuesPresent); } get type() { @@ -23,7 +23,7 @@ export class PolygonRecord extends BaseRingedRecord { static fromPresetReader(reader: ShapeReader, header: GeomHeader) { const hasZ = reader.hasZ; - const hasM = reader.hasM; + const hasOptionalM = reader.hasOptionalM; const shpStream = reader.shpStream; let z, m; @@ -33,10 +33,13 @@ export class PolygonRecord extends BaseRingedRecord { const parts = shpStream.readInt32Array(numParts, true); const xy = shpStream.readDoubleArray(numPoints * 2, true); if (hasZ) z = PolygonRecord.getZValues(shpStream, numPoints); + + const hasM = !this.recordReadingFinalized(shpStream, header) && hasOptionalM; if (hasM) m = PolygonRecord.getMValues(shpStream, numPoints); + const coords = PolygonRecord.getCoords(parts, xy, z, m); const polygons = assemblePolygonsWithHoles(coords); - return new PolygonRecord(polygons as PolygonCoord, GeomUtil.coordType(header.type)); + return new PolygonRecord(polygons as PolygonCoord, GeomUtil.coordType(header.type), hasM); } toGeoJson(): GeoJsonGeom { diff --git a/shpts/geometry/polyline.ts b/shpts/geometry/polyline.ts index fa376ef..228eff8 100644 --- a/shpts/geometry/polyline.ts +++ b/shpts/geometry/polyline.ts @@ -6,8 +6,8 @@ import { GeomUtil } from '@shpts/utils/geometry'; import { GeomHeader } from '@shpts/types/data'; export class PolyLineRecord extends BaseRingedRecord { - constructor(public coords: PolyLineCoord, coordType: CoordType) { - super(coordType); + constructor(public coords: PolyLineCoord, coordType: CoordType, hasMValuesPresent: boolean) { + super(coordType, hasMValuesPresent); } get type() { @@ -17,7 +17,7 @@ export class PolyLineRecord extends BaseRingedRecord { static fromPresetReader(reader: ShapeReader, header: GeomHeader) { const hasZ = reader.hasZ; - const hasM = reader.hasM; + const hasOptionalM = reader.hasOptionalM; const shpStream = reader.shpStream; let z, m; @@ -27,9 +27,12 @@ export class PolyLineRecord extends BaseRingedRecord { const parts = shpStream.readInt32Array(numParts, true); const xy = shpStream.readDoubleArray(numPoints * 2, true); if (hasZ) z = PolyLineRecord.getZValues(shpStream, numPoints); + + const hasM = !this.recordReadingFinalized(shpStream, header) && hasOptionalM; if (hasM) m = PolyLineRecord.getMValues(shpStream, numPoints); + const coords = PolyLineRecord.getCoords(parts, xy, z, m); - return new PolyLineRecord(coords as PolyLineCoord, GeomUtil.coordType(header.type)); + return new PolyLineRecord(coords as PolyLineCoord, GeomUtil.coordType(header.type), hasM); } toGeoJson() { diff --git a/shpts/reader/shpReader.ts b/shpts/reader/shpReader.ts index de7c03a..90ac78d 100644 --- a/shpts/reader/shpReader.ts +++ b/shpts/reader/shpReader.ts @@ -8,6 +8,13 @@ import { BoundingBox, GeomHeader, PartsInfo, ShpHeader } from '@shpts/types/data import { MemoryStream } from '@shpts/utils/stream'; import { MultiPatchRecord } from '@shpts/geometry/multipatch'; +// According to Shape spec, M values less than this is NaN +export const mNaN = -Math.pow(-10, 38); + +function isMNaN(m: number) { + return m <= mNaN; +} + export class ShapeReader { private shxStream: MemoryStream; private shxHeader: ShpHeader; @@ -16,7 +23,7 @@ export class ShapeReader { readonly recordCount: number = 0; readonly hasZ: boolean; - readonly hasM: boolean; + readonly hasOptionalM: boolean; private constructor(shp: ArrayBuffer, shx: ArrayBuffer) { this.shpStream = new MemoryStream(shp); @@ -27,9 +34,12 @@ export class ShapeReader { if (this.shpHeader.type !== this.shxHeader.type) throw new Error('SHP / SHX shapetype mismatch'); + //const zRangeDefined = !isMNaN(this.shpHeader.zRange.min) && !isMNaN(this.shpHeader.zRange.max); + //const mRangeDefined = !isMNaN(this.shpHeader.mRange.min) && !isMNaN(this.shpHeader.mRange.max); + this.recordCount = (this.shxHeader.fileLength - 100) / 8; this.hasZ = GeomUtil.hasZ(this.shpHeader.type); - this.hasM = GeomUtil.hasM(this.shpHeader.type); + this.hasOptionalM = GeomUtil.hasOptionalM(this.shpHeader.type); } static async fromFile(shp: File, shx: File) { @@ -53,19 +63,27 @@ export class ShapeReader { const shpType = stream.seek(32).readInt32(true); stream.seek(36); const extent = this.readBbox(stream); + const zMin = stream.readDouble(true); + const zMax = stream.readDouble(true); + const mMin = stream.readDouble(true); + const mMax = stream.readDouble(true); const result = { type: shpType as ShapeType, fileLength: fileLen * 2, extent: extent, + zRange: { min: zMin, max: zMax }, + mRange: { min: mMin, max: mMax }, }; return result; } private readGeomHeader(): GeomHeader { + const offset = this.shpStream.tell; const recNum = this.shpStream.readInt32(false); const len = this.shpStream.readInt32(false); const type: ShapeType = this.shpStream.readInt32(true) as ShapeType; return { + offset, length: len, recordNum: recNum, type: type, diff --git a/shpts/shpts.ts b/shpts/shpts.ts index f7740ec..98b88f1 100644 --- a/shpts/shpts.ts +++ b/shpts/shpts.ts @@ -13,6 +13,7 @@ import { DbfFieldDescr, DbfFieldType } from './types/dbfTypes'; import { Coord, CoordType } from './types/coordinate'; import { triangulate } from './utils/triangulation'; import { BaseRecord } from './geometry/base'; +import { ShapeType } from './utils/geometry'; export { DbfReader, @@ -31,4 +32,4 @@ export { triangulate, }; -export type { DbfFieldType, DbfFieldDescr, Coord }; +export type { DbfFieldType, DbfFieldDescr, Coord, ShapeType }; diff --git a/shpts/types/data.ts b/shpts/types/data.ts index 6c774d1..ce2090e 100644 --- a/shpts/types/data.ts +++ b/shpts/types/data.ts @@ -11,6 +11,8 @@ export interface ShpHeader { readonly extent: BoundingBox; readonly type: ShapeType; readonly fileLength: number; + readonly zRange: { min: number; max: number }; + readonly mRange: { min: number; max: number }; } export interface ShxRecord { @@ -19,6 +21,7 @@ export interface ShxRecord { } export interface GeomHeader { + offset: number; recordNum: number; length: number; type: ShapeType; diff --git a/shpts/utils/geometry.ts b/shpts/utils/geometry.ts index 7623fb8..1dc30e2 100644 --- a/shpts/utils/geometry.ts +++ b/shpts/utils/geometry.ts @@ -41,7 +41,7 @@ export class GeomUtil { return type === CoordType.XYZM; } - static hasM(shapeType: ShapeType): boolean { + static hasOptionalM(shapeType: ShapeType): boolean { const type = GeomUtil.coordType(shapeType); return type === CoordType.XYZM || type === CoordType.XYM; } diff --git a/test/polygon.test.ts b/test/polygon.test.ts index 78eca65..432772f 100644 --- a/test/polygon.test.ts +++ b/test/polygon.test.ts @@ -1,6 +1,6 @@ import { expect, test } from 'vitest'; import { expectGeometry, expectRing, openFileAsArray } from './utils'; -import { ShapeReader, PolygonRecord, CoordType } from '@shpts/shpts'; +import { ShapeReader, PolygonRecord, CoordType, FeatureReader } from '@shpts/shpts'; test('Reading PolygonRecord', async () => { const shpBuffer = openFileAsArray('testdata/polygon.shp'); @@ -39,6 +39,7 @@ test('Reading PolygonRecord', async () => { ]); geom = expectGeometry(reader, 1, CoordType.XY, PolygonRecord); + expect(geom.hasM).toBeFalsy(); expect(geom.type).toEqual('Polygon'); expect(geom.coords.length).toBe(1); polygon = geom.coords[0]; @@ -114,6 +115,7 @@ test('Reading PolygonRecord with M', async () => { expect(reader.recordCount).toBe(2); let geom = expectGeometry(reader, 0, CoordType.XYM, PolygonRecord); + expect(geom.hasM).toBeTruthy(); expect(geom.type).toEqual('Polygon'); expect(geom.coords.length).toBe(1); let polygon = geom.coords[0]; @@ -256,3 +258,19 @@ test('Reading PolygonZ Terrain Example', async () => { const reader = await ShapeReader.fromArrayBuffer(shpBuffer, shxBuffer); expect(reader.recordCount).toBe(42798); }); + +test('Reading PolygonZ without M values', async () => { + const shpBuffer = openFileAsArray('testdata/polygonZ.shp'); + const shxBuffer = openFileAsArray('testdata/polygonZ.shx'); + const dbfBuffer = openFileAsArray('testdata/polygonZ.dbf'); + + const reader = await FeatureReader.fromArrayBuffers(shpBuffer, shxBuffer, dbfBuffer); + const col = reader.readFeatureCollection(); + const features = col.features; + expect(features.length).toBe(1); + + const geom = features[0].geom; + expect(geom.type).toEqual('Polygon'); + expect(geom.hasM).toBeFalsy(); + expect(geom.hasZ).toBeTruthy(); +}); diff --git a/testdata/polygonZ.cpg b/testdata/polygonZ.cpg new file mode 100644 index 0000000..3ad133c --- /dev/null +++ b/testdata/polygonZ.cpg @@ -0,0 +1 @@ +UTF-8 \ No newline at end of file diff --git a/testdata/polygonZ.dbf b/testdata/polygonZ.dbf new file mode 100644 index 0000000000000000000000000000000000000000..ddd1ef40f0cbb9bda5722354a075fe8c282c4b4a GIT binary patch literal 78 mcmZRsWn^Y#U|?`$-~p1Dz|GSICg=xZaKm^|npXh<45R>hdjqxr literal 0 HcmV?d00001 diff --git a/testdata/polygonZ.prj b/testdata/polygonZ.prj new file mode 100644 index 0000000..5c6f76d --- /dev/null +++ b/testdata/polygonZ.prj @@ -0,0 +1 @@ +PROJCS["WGS_1984_Web_Mercator_Auxiliary_Sphere",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Mercator_Auxiliary_Sphere"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",0.0],PARAMETER["Standard_Parallel_1",0.0],PARAMETER["Auxiliary_Sphere_Type",0.0],UNIT["Meter",1.0]] \ No newline at end of file diff --git a/testdata/polygonZ.shp b/testdata/polygonZ.shp new file mode 100644 index 0000000000000000000000000000000000000000..91063ff87266a4841b263ff0a622b05d9ee1ff93 GIT binary patch literal 588 zcmZQzQ0HR63K-R1Ff%al1LamsSSB(rB;8T#epmdb$xe2I$wcOPa zt9DdDMj-bCF=hd+WDtbvWdqXOK%C&ccIM*4>2UjC_OthRn3e{+ILh9dGdIB?1Ma@O zAFq_BmZUq1?f&;9X18~`#z_s@E{^EtS@$cd-kFo`SiSE^A9s(7W6%xTdY9sK z$Mf>0lPf Date: Wed, 8 Jan 2025 20:17:24 +0000 Subject: [PATCH 2/3] feat: remove parsing unused ranges --- shpts/reader/shpReader.ts | 6 ------ shpts/types/data.ts | 2 -- 2 files changed, 8 deletions(-) diff --git a/shpts/reader/shpReader.ts b/shpts/reader/shpReader.ts index 90ac78d..010cdbb 100644 --- a/shpts/reader/shpReader.ts +++ b/shpts/reader/shpReader.ts @@ -63,16 +63,10 @@ export class ShapeReader { const shpType = stream.seek(32).readInt32(true); stream.seek(36); const extent = this.readBbox(stream); - const zMin = stream.readDouble(true); - const zMax = stream.readDouble(true); - const mMin = stream.readDouble(true); - const mMax = stream.readDouble(true); const result = { type: shpType as ShapeType, fileLength: fileLen * 2, extent: extent, - zRange: { min: zMin, max: zMax }, - mRange: { min: mMin, max: mMax }, }; return result; } diff --git a/shpts/types/data.ts b/shpts/types/data.ts index ce2090e..71e35ac 100644 --- a/shpts/types/data.ts +++ b/shpts/types/data.ts @@ -11,8 +11,6 @@ export interface ShpHeader { readonly extent: BoundingBox; readonly type: ShapeType; readonly fileLength: number; - readonly zRange: { min: number; max: number }; - readonly mRange: { min: number; max: number }; } export interface ShxRecord { From 6de72d011bf10f41f2432fdecf4552bb34f30fd3 Mon Sep 17 00:00:00 2001 From: Vojta Tomas Date: Wed, 8 Jan 2025 20:20:54 +0000 Subject: [PATCH 3/3] feat: remove unused NaN value --- shpts/reader/shpReader.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/shpts/reader/shpReader.ts b/shpts/reader/shpReader.ts index 010cdbb..d523687 100644 --- a/shpts/reader/shpReader.ts +++ b/shpts/reader/shpReader.ts @@ -8,13 +8,6 @@ import { BoundingBox, GeomHeader, PartsInfo, ShpHeader } from '@shpts/types/data import { MemoryStream } from '@shpts/utils/stream'; import { MultiPatchRecord } from '@shpts/geometry/multipatch'; -// According to Shape spec, M values less than this is NaN -export const mNaN = -Math.pow(-10, 38); - -function isMNaN(m: number) { - return m <= mNaN; -} - export class ShapeReader { private shxStream: MemoryStream; private shxHeader: ShpHeader;