feat(geoarrow): New module for geoarrow processing (#3149)

modules/arrow/src/triangulate-on-worker.ts

@@ -5,7 +5,7 @@
 import * as arrow from 'apache-arrow';
 import type {WorkerOptions} from '@loaders.gl/worker-utils';
 import {processOnWorker} from '@loaders.gl/worker-utils';
-import type {GeoArrowEncoding} from '@loaders.gl/schema';
+import type {GeoArrowEncoding} from '@loaders.gl/geoarrow';
 import {BinaryDataFromGeoArrow} from '@loaders.gl/gis';
 
 // __VERSION__ is injected by babel-plugin-version-inline

modules/arrow/tsconfig.json

@@ -10,6 +10,7 @@
   "references": [
     {"path": "../core"},
     {"path": "../gis"},
+    {"path": "../geoarrow"},
     {"path": "../loader-utils"},
     {"path": "../schema"},
     {"path": "../schema-utils"},

modules/geoarrow/README.md

@@ -0,0 +1,7 @@
+# @loaders.gl/geoarrow
+
+This module contains support for the Apache Arrow GeoArrow format.
+
+[loaders.gl](https://loaders.gl/docs) is a collection of loaders for big data visualizations.
+
+For documentation please visit the [website](https://loaders.gl).

modules/geoarrow/bundle.ts

@@ -0,0 +1,3 @@
+// Re-export core API so they don't get overwritten
+export * from '@loaders.gl/core';
+export * from '@loaders.gl/geoarrow';

modules/geoarrow/package.json

@@ -0,0 +1,63 @@
+{
+  "name": "@loaders.gl/geoarrow",
+  "version": "4.4.0-alpha.0",
+  "description": "GeoArrow columnar geometry encoding and decoding",
+  "license": "MIT",
+  "type": "module",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/visgl/loaders.gl"
+  },
+  "keywords": [
+    "loader",
+    "parser",
+    "writer",
+    "encoder",
+    "geoarrow",
+    "apache-arrow",
+    "arrow",
+    "binary columnar",
+    "cloud native",
+    "webgl",
+    "webgpu"
+  ],
+  "types": "dist/index.d.ts",
+  "main": "dist/index.cjs",
+  "module": "dist/index.js",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    },
+    "./exports/*": {
+      "types": "./dist/exports/*.d.ts",
+      "import": "./dist/exports/*.js"
+    }
+  },
+  "sideEffects": false,
+  "files": [
+    "src",
+    "dist",
+    "README.md"
+  ],
+  "browser": {
+    "fs": false
+  },
+  "scripts": {
+    "pre-build": "",
+    "build-bundle": "ocular-bundle ./bundle.ts --output=dist/dist.min.js",
+    "build-bundle-dev": "ocular-bundle ./bundle.ts --env=dev --output=dist/dist.dev.js"
+  },
+  "dependencies": {
+    "@math.gl/polygon": "^4.1.0",
+    "apache-arrow": ">= 16.1.0"
+  },
+  "peerDependencies": {
+    "@loaders.gl/core": "4.4.0-alpha.0"
+  },
+  "gitHead": "3213679d79e6ff2814d48fd3337acfa446c74099"
+}

modules/geoarrow/src/geo-column/geo-column-info.ts

@@ -0,0 +1,114 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// import type {GeoColumn} from './geo-column';
+// import {getGeoParquetMetadata, GeoParquetMetadata} from '../metadata/geoparquet-metadata';
+// import {getGeoArrowMetadata, GeoArrowMetadata} from '../metadata_3/geoarrow-metadata';
+
+/**
+ * Information about a binary geometry
+ */
+export type geoColumnInfo = {
+  type: 'Point' | 'LineString' | 'Polygon';
+  /** The GeoJSON style geometry type corresponding to this particular binary geometry */
+  multiGeometryType:
+    | 'Point'
+    | 'LineString'
+    | 'Polygon'
+    | 'MultiPoint'
+    | 'MultiLineString'
+    | 'MultiPolygon';
+  /** Is this a "Multi" version of the binary geometry? */
+  isMultiGeometry: boolean;
+  /** How many dimensions are the coordinates? */
+  dimension: number;
+  /** How many points does this geometry have? */
+  pointCount: number;
+  /** How many coordinates does this geometry have? */
+  coordinateCount: number;
+};
+
+/**
+ * @returns information about a binary geometry
+ */
+// export function getGeoColumnInfo(geometry: GeoColumn): GeoColumnInfo {
+
+// }
+
+// // loaders.gl
+// // SPDX-License-Identifier: MIT
+// // Copyright (c) vis.gl contributors
+
+// import type {Schema, Field, GeoArrowMetadata, GeoArrowEncoding, Geometry} from '@loaders.gl/schema';
+// import {convertArrowToSchema} from '@loaders.gl/schema-utils';
+// import * as arrow from 'apache-arrow';
+// import {getGeometryColumnsFromSchema, getGeometryMetadataForField} from '../geoarrow/geoarrow-metadata';
+// import { getGeoMetadata as getParquetMe} from './geoparquet-metadata';
+
+// /**
+// * A geoarrow / geoparquet geo metadata object
+// * (stored in stringified form in the top level metadata 'geo' key)
+// * @see https://github.com/opengeospatial/geoparquet/blob/main/format-specs/geoparquet.md
+// * @see https://github.com/geoarrow/geoarrow
+// * */
+// export type GeoTableMetadata = {
+//  version?: string;
+//  primaryGeometryColumn?: string;
+//  columns: Record<string, GeoColumnInfo>;
+//  [key: string]: unknown;
+// };
+
+// /** A geoarrow / geoparquet geo metadata for one geometry column  */
+// export type GeoColumnInfo = {
+//  encoding: 'wkb' | 'wkt' | 'none';
+//  geometryTypes: Geometry['type'][];
+//  dimension: number;
+//  crs?: object | null;
+//  orientation?: 'counterclockwise';
+//  bbox?: [number, number, number, number] | [number, number, number, number, number, number];
+//  edges?: 'planar' | 'spherical';
+//  epoch?: number;
+
+//  [key: string]: unknown;
+//  // geoParquetGeometryType:
+// };
+
+// export type GeoArrowInfo = {
+//   geometryColumns: Record<string, GeometryColumnInfo>
+// };
+
+// /**
+//  * get geometry columns from arrow table
+//  */
+// export function getGeoArrowInfo(arrowTable: arrow.Table): GeoArrowInfo {
+//  const schema = convertArrowToSchema(arrowTable.schema);
+//   const geometryColumns = getGeometryColumnsFromSchema(schema);
+
+//   // get encoding from geometryColumns['geometry']
+//   const encoding = geometryColumns.geometry.encoding;
+
+//   // Remove geometry columns
+//   const propertyColumnNames = arrowTable.schema.fields
+//     .map((field) => field.name)
+//     // TODO - this deletes all geometry columns
+//     .filter((name) => name in geometryColumns);
+//   const propertiesTable = arrowTable.select(propertyColumnNames);
+
+//   const arrowGeometryColumn = arrowTable.getChild('geometry');
+
+//   for (let row = 0; row < arrowTable.numRows; row++) {
+//     // get the geometry value from arrow geometry column
+//     // Note that type can vary
+//     const arrowGeometry = arrowGeometryColumn?.get(row);
+//     // parse arrow geometry to geojson feature
+//     const feature = convertGeoArrowGeometryToGeoJSON(arrowGeometry, encoding);
+//     if (feature) {
+//       const properties = propertiesTable.get(row)?.toJSON() || {};
+//       features.push({type: 'Feature', geometry: feature, properties});
+//     }
+//   }
+
+//   return {
+//   };
+// }

modules/geoarrow/src/geo-column/geo-column.ts

@@ -0,0 +1,85 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// GIS
+import type {TypedArray} from '@math.gl/types';
+
+// BINARY FORMAT GEOMETRY
+
+export type GeoColumnType = 'Point' | 'LineString' | 'Polygon';
+
+/** A geometry column can be built from list of geometry column data chunks */
+export type GeoColumn<MetadataT = Record<string, unknown>> = {
+  shape: 'geo-column';
+  metadata?: MetadataT;
+  data: GeoColumnData[];
+};
+
+export type GeoColumnData = GeoColumnPointData | GeoColumnLineData | GeoColumnPolygonData;
+
+/**
+ * Describes a single contiguous chunk of geometries in binary columnar format
+ * Extracts the nested position and offset arrays from a GeoArrow column
+ * @note Designed to be a cheap operation: does not create any new arrays, just holds extracted references the existing arrays
+ */
+export type GeoColumnCommonData = {
+  shape: 'geo-column-data';
+
+  /** Number of rows geometries (can be less than pointCount if representing multipoints) */
+  numRows: number;
+  /** Offset to the end of the next geometry (length = rowCount) */
+  rowOffsets: Uint32Array;
+
+  /** Number of positions */
+  numPositions: number;
+  /** Data for coordinates, either interleaved or separate arrays */
+  positions: GeoColumnPositions;
+
+  /** Stores the data for the respective geometry types */
+  // points?: GeoColumnPointData;
+  // lines?: GeoColumnLineData;
+  // polygons?: GeoColumnPolygonData;
+};
+
+/** Columnar point geometry: an array of positions */
+export type GeoColumnPointData = GeoColumnCommonData & {
+  type: 'Point';
+};
+
+/** Columnar line geometry, array of positions and indices to the start of each line */
+export type GeoColumnLineData = GeoColumnCommonData & {
+  type: 'LineString';
+
+  /** Offset to the next path within the multi feature */
+  pathOffsets: Uint32Array;
+};
+
+/** Columnar polygon geometry, an array of positions to each primitite polygon and polygon */
+export type GeoColumnPolygonData = GeoColumnCommonData & {
+  type: 'Polygon';
+  /** Offset to next polygon */
+  polygonOffsets: Uint32Array;
+  /** Offset to next primitive polygon */
+  ringOffsets: Uint32Array;
+};
+
+export type GeoColumnPositions = GeoColumnInterleavedPositions | GeoColumnSeparatePositions;
+
+/** Positions are in a single coordinate array */
+export type GeoColumnInterleavedPositions = {
+  layout: 'interleaved';
+  /** Dimension, i.e. number of coordinates per position: 2, 3 or 4 */
+  stride: 2 | 3 | 4;
+  /** Flat array of position coordinates (length = positionCount * positionStride */
+  coordinates: TypedArray;
+};
+
+/** Positions are in separate coordinate arrays */
+export type GeoColumnSeparatePositions = {
+  layout: 'separate';
+  /** Dimension, i.e. number of coordinates per position: 2, 3 or 4 */
+  stride: 2 | 3 | 4;
+  /** Flat array of position coordinates (length = positionCount * positionStride */
+  coordinates: TypedArray[];
+};