feat(geo): add geography data type

Cargo.toml

@@ -28,6 +28,7 @@ members = [
  "src/common/vector",
  "src/common/license",
  "src/common/parquet2",
+ "src/common/geo",
  "src/query/ast",
  "src/query/async_functions",
  "src/query/codegen",
@@ -124,6 +125,7 @@ databend-common-exception = { path = "src/common/exception" }
 databend-common-expression = { path = "src/query/expression" }
 databend-common-formats = { path = "src/query/formats" }
 databend-common-functions = { path = "src/query/functions" }
+databend-common-geo = { path = "src/common/geo" }
 databend-common-grpc = { path = "src/common/grpc" }
 databend-common-hashtable = { path = "src/common/hashtable" }
 databend-common-http = { path = "src/common/http" }

src/common/geo/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "databend-common-geo"
+version = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+publish = { workspace = true }
+edition = { workspace = true }
+
+[lib]
+doctest = false
+test = true
+
+[dependencies]
+scroll = "0.12.0"
+
+[lints]
+workspace = true

src/common/geo/src/lib.rs

@@ -0,0 +1,16 @@
+// Copyright 2021 Datafuse Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod wkb;
+pub use wkb::read_wkb_header;

src/common/geo/src/wkb.rs

@@ -0,0 +1,94 @@
+// Copyright 2021 Datafuse Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::io::Cursor;
+
+use scroll::Endian;
+use scroll::IOread;
+use scroll::IOwrite;
+
+const WKB_XDR: u8 = 0;
+const WKB_NDR: u8 = 1;
+const WKB_Z: u32 = 0x80000000;
+const WKB_M: u32 = 0x40000000;
+const WKB_SRID: u32 = 0x20000000;
+const WKB_TYPE_MASK: u32 = 7;
+
+pub fn read_wkb_header(mut raw: &[u8]) -> Result<WkbInfo, String> {
+ let byte_order = raw.ioread::<u8>().map_err(|e| e.to_string())?;
+ let endian = match byte_order {
+ WKB_XDR => Endian::from(false),
+ WKB_NDR => Endian::from(true),
+ _ => return Err("Unexpected byte order.".to_string()),
+ };
+
+ let type_id = raw.ioread_with::<u32>(endian).map_err(|e| e.to_string())?;
+
+ let srid = if type_id & WKB_SRID != 0 {
+ Some(raw.ioread::<i32>().map_err(|e| e.to_string())?)
+ } else {
+ None
+ };
+
+ let base_type = match type_id & WKB_TYPE_MASK {
+ 1 => WkbType::Point,
+ 2 => WkbType::LineString,
+ 3 => WkbType::Polygon,
+ 4 => WkbType::MultiPoint,
+ 5 => WkbType::MultiLineString,
+ 6 => WkbType::MultiPolygon,
+ 7 => WkbType::GeometryCollection,
+ _ => return Err("Unknown Geometry Type".to_string()),
+ };
+
+ Ok(WkbInfo {
+ endian,
+ base_type,
+ has_z: type_id & WKB_Z != 0,
+ has_m: type_id & WKB_M != 0,
+ srid,
+ })
+}
+
+#[derive(Debug)]
+pub enum WkbType {
+ Point = 1,
+ LineString = 2,
+ Polygon = 3,
+ MultiPoint = 4,
+ MultiLineString = 5,
+ MultiPolygon = 6,
+ GeometryCollection = 7,
+}
+
+#[derive(Debug)]
+pub struct WkbInfo {
+ pub endian: Endian,
+ pub base_type: WkbType,
+ pub has_z: bool,
+ pub has_m: bool,
+ pub srid: Option<i32>,
+}
+
+pub const POINT_SIZE: usize = 21;
+
+pub fn make_point(x: f64, y: f64) -> Vec<u8> {
+ let bytes = Vec::with_capacity(21);
+ let mut bytes = Cursor::new(bytes);
+ bytes.iowrite(WKB_NDR).unwrap();
+ bytes.iowrite(WkbType::Point as u32).unwrap();
+ bytes.iowrite(x).unwrap();
+ bytes.iowrite(y).unwrap();
+ bytes.into_inner()
+}

src/meta/proto-conv/src/schema_from_to_protobuf_impl.rs

@@ -258,6 +258,7 @@ impl FromToProto for ex::TableDataType {
  }
  Dt24::VariantT(_) => ex::TableDataType::Variant,
  Dt24::GeometryT(_) => ex::TableDataType::Geometry,
+ Dt24::GeographyT(_) => ex::TableDataType::Geography,
  Dt24::DecimalT(x) => {
  ex::TableDataType::Decimal(ex::types::decimal::DecimalDataType::from_pb(x)?)
  }
@@ -324,6 +325,7 @@ impl FromToProto for ex::TableDataType {
  }
  TableDataType::Variant => new_pb_dt24(Dt24::VariantT(pb::Empty {})),
  TableDataType::Geometry => new_pb_dt24(Dt24::GeometryT(pb::Empty {})),
+ TableDataType::Geography => new_pb_dt24(Dt24::GeographyT(pb::Empty {})),
  };
  Ok(x)
  }

src/meta/proto-conv/src/util.rs

@@ -136,6 +136,7 @@ const META_CHANGE_LOG: &[(u64, &str)] = &[
  (104, "2024-08-02: Add: add share catalog into Catalog meta"),
  (105, "2024-08-05: Add: add Dictionary meta"),
  (106, "2024-08-08: Add: add QueryTokenInfo"),
+ (107, "2024-08-09: Add: datatype.proto/DataType Geography type")
  // Dear developer:
  // If you're gonna add a new metadata version, you'll have to add a test for it.
  // You could just copy an existing test file(e.g., `../tests/it/v024_table_meta.rs`)

src/meta/proto-conv/tests/it/main.rs

@@ -110,3 +110,4 @@ mod v103_share_meta_v2;
 mod v104_share_catalog;
 mod v105_dictionary_meta;
 mod v106_query_token;
+mod v107_geography_datatype;