Geo Spatial: 1. geography common

cmake/nebula/GeneralCompilerConfig.cmake

@@ -23,6 +23,7 @@ add_compile_options(-Wignored-qualifiers)
 
 # For s2
 add_definitions(-DS2_USE_GLOG)
+add_definitions(-DS2_USE_GFLAGS)
 # For breakpad
 add_definitions(-D__STDC_FORMAT_MACROS)
 

src/codec/RowReaderV2.cpp

@@ -175,6 +175,10 @@ Value RowReaderV2::getValueByIndex(const int64_t index) const noexcept {
  dt.microsec = microsec;
  return dt;
  }
+ case meta::cpp2::PropertyType::GEOGRAPHY: {
+ // TODO(jie)
+ return Geography();
+ }
  case meta::cpp2::PropertyType::UNKNOWN:
  break;
  }

src/codec/RowWriterV2.cpp

@@ -126,6 +126,9 @@ RowWriterV2::RowWriterV2(RowReader& reader) : RowWriterV2(reader.getSchema()) {
  case Value::Type::DATETIME:
  set(i, v.moveDateTime());
  break;
+ case Value::Type::GEOGRAPHY:
+ // TODO(jie)
+ break;
  default:
  LOG(FATAL) << "Invalid data: " << v << ", type: " << v.typeName();
  }
@@ -203,6 +206,9 @@ WriteResult RowWriterV2::setValue(ssize_t index, const Value& val) noexcept {
  return write(index, val.getTime());
  case Value::Type::DATETIME:
  return write(index, val.getDateTime());
+ case Value::Type::GEOGRAPHY:
+ // TODO(jie)
+ return WriteResult::TYPE_MISMATCH;
  default:
  return WriteResult::TYPE_MISMATCH;
  }
@@ -637,6 +643,7 @@ WriteResult RowWriterV2::write(ssize_t index, folly::StringPiece v) noexcept {
  auto field = schema_->field(index);
  auto offset = headerLen_ + numNullBytes_ + field->offset();
  switch (field->type()) {
+ case meta::cpp2::PropertyType::GEOGRAPHY: // write wkb
  case meta::cpp2::PropertyType::STRING: {
  if (isSet_[index]) {
  // The string value has already been set, we need to turn it
@@ -794,6 +801,9 @@ WriteResult RowWriterV2::checkUnsetFields() noexcept {
  case Value::Type::DATETIME:
  r = write(i, defVal.getDateTime());
  break;
+ case Value::Type::GEOGRAPHY:
+ // TODO(jie)
+ break;
  default:
  LOG(FATAL) << "Unsupported default value type: " << defVal.typeName()
  << ", default value: " << defVal

src/codec/test/CMakeLists.txt

@@ -28,6 +28,7 @@ set(CODEC_TEST_LIBS
  $<TARGET_OBJECTS:process_obj>
  $<TARGET_OBJECTS:expression_obj>
  $<TARGET_OBJECTS:function_manager_obj>
+ $<TARGET_OBJECTS:wkt_wkb_io_obj>
  $<TARGET_OBJECTS:agg_function_manager_obj>
  $<TARGET_OBJECTS:time_utils_obj>
  $<TARGET_OBJECTS:version_obj>

src/codec/test/SchemaWriter.cpp

@@ -75,6 +75,9 @@ SchemaWriter& SchemaWriter::appendCol(folly::StringPiece name,
  case PropertyType::DATETIME:
  size = sizeof(int16_t) + 5 * sizeof(int8_t) + sizeof(int32_t);
  break;
+ case PropertyType::GEOGRAPHY:
+ size = 2 * sizeof(int32_t); // as same as STRING
+ break;
  default:
  LOG(FATAL) << "Unknown column type";
  }

src/common/CMakeLists.txt

@@ -26,3 +26,4 @@ nebula_add_subdirectory(graph)
 nebula_add_subdirectory(plugin)
 nebula_add_subdirectory(utils)
 nebula_add_subdirectory(ssl)
+nebula_add_subdirectory(geo)

src/common/base/Base.h

@@ -67,7 +67,7 @@
 
 #include "common/base/Logging.h"
 
-#define MUST_USE_RESULT __attribute__((warn_unused_result))
+#define NG_MUST_USE_RESULT __attribute__((warn_unused_result))
 #define DONT_OPTIMIZE __attribute__((optimize("O0")))
 
 #define ALWAYS_INLINE __attribute__((always_inline))

src/common/conf/Configuration.h

@@ -27,11 +27,11 @@ class Configuration final {
  /**
  * Parse from a file
  */
- Status MUST_USE_RESULT parseFromFile(const std::string &filename);
+ Status NG_MUST_USE_RESULT parseFromFile(const std::string &filename);
  /**
  * Parse from a string buffer
  */
- Status MUST_USE_RESULT parseFromString(const std::string &content);
+ Status NG_MUST_USE_RESULT parseFromString(const std::string &content);
 
  std::string dumpToString() const;
 
@@ -42,19 +42,20 @@ class Configuration final {
  * @key item key
  * @val to hold the item value.
  */
- Status MUST_USE_RESULT fetchAsInt(const char *key, int64_t &val) const;
- Status MUST_USE_RESULT fetchAsDouble(const char *key, double &val) const;
- Status MUST_USE_RESULT fetchAsBool(const char *key, bool &val) const;
- Status MUST_USE_RESULT fetchAsString(const char *key, std::string &val) const;
+ Status NG_MUST_USE_RESULT fetchAsInt(const char *key, int64_t &val) const;
+ Status NG_MUST_USE_RESULT fetchAsDouble(const char *key, double &val) const;
+ Status NG_MUST_USE_RESULT fetchAsBool(const char *key, bool &val) const;
+ Status NG_MUST_USE_RESULT fetchAsString(const char *key, std::string &val) const;
 
- Status MUST_USE_RESULT fetchAsIntArray(const char *key, std::vector<int64_t> &val) const;
- Status MUST_USE_RESULT fetchAsDoubleArray(const char *key, std::vector<double> &val) const;
- Status MUST_USE_RESULT fetchAsBoolArray(const char *key, std::vector<bool> &val) const;
- Status MUST_USE_RESULT fetchAsStringArray(const char *key, std::vector<std::string> &val) const;
+ Status NG_MUST_USE_RESULT fetchAsIntArray(const char *key, std::vector<int64_t> &val) const;
+ Status NG_MUST_USE_RESULT fetchAsDoubleArray(const char *key, std::vector<double> &val) const;
+ Status NG_MUST_USE_RESULT fetchAsBoolArray(const char *key, std::vector<bool> &val) const;
+ Status NG_MUST_USE_RESULT fetchAsStringArray(const char *key,
+ std::vector<std::string> &val) const;
 
- Status MUST_USE_RESULT fetchAsSubConf(const char *key, Configuration &val) const;
+ Status NG_MUST_USE_RESULT fetchAsSubConf(const char *key, Configuration &val) const;
 
- Status MUST_USE_RESULT upsertStringField(const char *key, const std::string &val);
+ Status NG_MUST_USE_RESULT upsertStringField(const char *key, const std::string &val);
 
  // Iterate through every key in the configuration
  Status forEachKey(std::function<void(const std::string &)> processor) const;

src/common/datatypes/CMakeLists.txt

@@ -14,6 +14,7 @@ nebula_add_library(
  Map.cpp
  List.cpp
  Set.cpp
+ Geography.cpp
 )
 
 nebula_add_subdirectory(test)

src/common/datatypes/CommonCpp2Ops.h

@@ -24,6 +24,7 @@ struct Map;
 struct Set;
 struct List;
 struct DataSet;
+struct Geography;
 } // namespace nebula
 
 namespace apache::thrift {
@@ -43,6 +44,7 @@ SPECIALIZE_CPP2OPS(nebula::Map);
 SPECIALIZE_CPP2OPS(nebula::Set);
 SPECIALIZE_CPP2OPS(nebula::List);
 SPECIALIZE_CPP2OPS(nebula::DataSet);
+SPECIALIZE_CPP2OPS(nebula::Geography);
 
 } // namespace apache::thrift
 

src/common/datatypes/Geography.cpp

@@ -0,0 +1,86 @@
+/* Copyright (c) 2020 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License,
+ * attached with Common Clause Condition 1.0, found in the LICENSES directory.
+ */
+
+#include "common/datatypes/Geography.h"
+
+#include <folly/String.h>
+#include <folly/hash/Hash.h>
+
+#include <cstdint>
+
+#include "common/geo/GeoUtils.h"
+#include "common/geo/io/wkb/WKBReader.h"
+#include "common/geo/io/wkb/WKBWriter.h"
+#include "common/geo/io/wkt/WKTReader.h"
+#include "common/geo/io/wkt/WKTWriter.h"
+
+namespace nebula {
+
+StatusOr<Geography> Geography::fromWKT(const std::string& wkt) {
+ auto geomRet = WKTReader().read(wkt);
+ NG_RETURN_IF_ERROR(geomRet);
+ auto geom = geomRet.value();
+ auto wkb = WKBWriter().write(geom);
+ return Geography(wkb);
+}
+
+GeoShape Geography::shape() const {
+ // TODO(jie) May store the shapetype as the data member of Geography is ok.
+ const uint8_t* beg = reinterpret_cast<const uint8_t*>(wkb.data());
+ const uint8_t* end = beg + wkb.size();
+ WKBReader reader;
+ auto byteOrderRet = reader.readByteOrder(beg, end);
+ if (!byteOrderRet.ok()) {
+ return GeoShape::UNKNOWN;
+ }
+ ByteOrder byteOrder = byteOrderRet.value();
+ auto shapeTypeRet = reader.readShapeType(beg, end, byteOrder);
+ if (!shapeTypeRet.ok()) {
+ return GeoShape::UNKNOWN;
+ }
+ return shapeTypeRet.value();
+}
+
+std::unique_ptr<std::string> Geography::asWKT() const {
+ auto geomRet = WKBReader().read(wkb);
+ if (!geomRet.ok()) {
+ LOG(ERROR) << geomRet.status();
+ return nullptr;
+ }
+ auto geom = geomRet.value();
+ return std::make_unique<std::string>(WKTWriter().write(geom));
+}
+
+std::unique_ptr<std::string> Geography::asWKBHex() const {
+ auto geomRet = WKBReader().read(wkb);
+ if (!geomRet.ok()) {
+ LOG(ERROR) << geomRet.status();
+ return nullptr;
+ }
+ auto geom = geomRet.value();
+ return std::make_unique<std::string>(folly::hexlify(WKBWriter().write(geom)));
+}
+
+std::unique_ptr<S2Region> Geography::asS2() const {
+ auto geomRet = WKBReader().read(wkb);
+ if (!geomRet.ok()) {
+ LOG(ERROR) << geomRet.status();
+ return nullptr;
+ }
+ auto geom = geomRet.value();
+ return GeoUtils::s2RegionFromGeomtry(geom);
+}
+
+} // namespace nebula
+
+namespace std {
+
+// Inject a customized hash function
+std::size_t hash<nebula::Geography>::operator()(const nebula::Geography& h) const noexcept {
+ return hash<std::string>{}(h.wkb);
+}
+
+} // namespace std

src/common/datatypes/Geography.h

@@ -0,0 +1,87 @@
+/* Copyright (c) 2020 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License,
+ * attached with Common Clause Condition 1.0, found in the LICENSES directory.
+ */
+
+#pragma once
+
+#include <s2/s2point.h>
+#include <s2/s2point_region.h>
+#include <s2/s2polyline.h>
+#include <s2/s2region.h>
+
+#include "common/base/StatusOr.h"
+#include "common/datatypes/Value.h"
+#include "common/geo/io/Geometry.h"
+
+// Do not include <s2/s2polygon.h> here, it will indirectly includes a header file which defines a
+// enum `BEGIN`(not enum class). While Geography.h is indirectly included by parser.yy, which has a
+// macro named `BEGIN`. So they will be conflicted.
+
+class S2Polygon;
+
+namespace nebula {
+
+// clang-format off
+/*
+static const std::unordered_map<GeoShape, S2Region> kShapeTypeToS2Region = {
+ // S2PointRegion is a wrapper of S2Point, and it inherits from the S2Region class while S2Point doesn't.
+ {GeoShape::POINT, S2PointRegion},
+ {GeoShape::LINESTRING, S2Polyline},
+ {GeoShape::POLYGON, S2Polygon},
+};
+*/
+// clang-format on
+
+// Do not construct a S2 object when constructing Geography. It's expensive.
+// We just construct S2 when doing computation.
+struct Geography {
+ std::string wkb; // TODO(jie) Is it better to store Geometry* or S2Region* here?
+
+ Geography() = default;
+
+ static StatusOr<Geography> fromWKT(const std::string& wkt);
+
+ GeoShape shape() const;
+
+ std::unique_ptr<std::string> asWKT() const;
+
+ std::unique_ptr<std::string> asWKBHex() const;
+
+ std::unique_ptr<S2Region> asS2() const;
+
+ std::string toString() const { return wkb; }
+
+ folly::dynamic toJson() const { return toString(); }
+
+ void clear() { wkb.clear(); }
+
+ void __clear() { clear(); }
+
+ bool operator==(const Geography& rhs) const { return wkb == rhs.wkb; }
+
+ bool operator!=(const Geography& rhs) const { return !(wkb == rhs.wkb); }
+
+ bool operator<(const Geography& rhs) const { return wkb < rhs.wkb; }
+
+ private:
+ explicit Geography(const std::string& bytes) {
+ // TODO(jie): Must ensure the bytes is valid
+ wkb = bytes;
+ }
+};
+
+inline std::ostream& operator<<(std::ostream& os, const Geography& g) { return os << g.wkb; }
+
+} // namespace nebula
+
+namespace std {
+
+// Inject a customized hash function
+template <>
+struct hash<nebula::Geography> {
+ std::size_t operator()(const nebula::Geography& h) const noexcept;
+};
+
+} // namespace std