Integrate Harffbuzz freetype to render Khmer Burmese and hindi

.gitmodules

@@ -40,6 +40,12 @@
 [submodule "platform/windows/vendor/vcpkg"]
 	path = platform/windows/vendor/vcpkg
 	url = https://github.com/microsoft/vcpkg.git
+[submodule "vendor/harfbuzzFreetype/freetype"]
+	path = vendor/freetype
+  url=https://github.com/freetype/freetype.git
+[submodule "vendor/harfbuzzFreetype/harfbuzz"]
+	path = vendor/harfbuzz
+  url=https://github.com/harfbuzz/harfbuzz
 [submodule "vendor/boost"]
 	path = vendor/boost
 	url = https://github.com/maplibre/maplibre-gl-native-boost.git

BUILD.bazel

@@ -18,6 +18,7 @@ load(
     "MLN_OPENGL_SOURCE",
     "MLN_PRIVATE_GENERATED_STYLE_HEADERS",
     "MLN_PUBLIC_GENERATED_STYLE_HEADERS",
+    "MLN_SHAPING_HARFBUZZ_SRCS",
 )
 load("//bazel:flags.bzl", "CPP_FLAGS", "MAPLIBRE_FLAGS")
 
@@ -79,6 +80,9 @@ cc_library(
         ":legacy_renderer": MLN_OPENGL_SOURCE,
         ":metal_renderer": MLN_DRAWABLES_SOURCE + MLN_DRAWABLES_MTL_SOURCE,
         "//conditions:default": [],
+    }) + select({
+        "//:harfbuzz_text_shaping": MLN_SHAPING_HARFBUZZ_SRCS,
+        "//conditions:default": [],
     }),
     hdrs = MLN_CORE_HEADERS + select({
         ":drawable_renderer": MLN_OPENGL_HEADERS + MLN_DRAWABLES_HEADERS + MLN_DRAWABLES_GL_HEADERS,
@@ -106,6 +110,9 @@ cc_library(
     }) + select({
         "@platforms//os:ios": ["GLES_SILENCE_DEPRECATION=1"],
         "//conditions:default": [],
+    }) + select({
+        "//:harfbuzz_text_shaping": ["MLN_TEXT_SHAPING_HARFBUZZ=1"],
+        "//conditions:default": [],
     }),
     includes = [
         "include",
@@ -115,6 +122,12 @@ cc_library(
             "vendor/metal-cpp",
         ],
         "//conditions:default": [],
+    }) + select({
+        "//:harfbuzz_text_shaping": [
+            "vendor/freetype/include",
+            "vendor/harfbuzz/src",
+        ],
+        "//conditions:default": [],
     }),
     local_defines = [
         r"MLN_VERSION_REV=\"standalone\"",
@@ -148,6 +161,12 @@ cc_library(
             "//vendor:metal-cpp",
         ],
         "//conditions:default": [],
+    }) + select({
+        ":harfbuzz_text_shaping": [
+            "//vendor:freetype",
+            "//vendor:harfbuzz",
+        ],
+        "//conditions:default": [],
     }) + select({
         ":rust": [
             "//rustutils:rustutilslib",
@@ -201,6 +220,31 @@ config_setting(
     },
 )
 
+# Selects the shaping implementation to utilize in the core
+
+string_flag(
+    name = "shaping",
+    build_setting_default = "harfbuzz",
+    values = [
+        "legacy",
+        "harfbuzz",
+    ],
+)
+
+config_setting(
+    name = "harfbuzz_text_shaping",
+    flag_values = {
+        ":shaping": "harfbuzz",
+    },
+)
+
+config_setting(
+    name = "legacy_text_shaping",
+    flag_values = {
+        ":shaping": "legacy",
+    },
+)
+
 bool_flag(
     name = "use_rust",
     build_setting_default = False,

CMakeLists.txt

@@ -18,6 +18,7 @@ option(MLN_DRAWABLE_RENDERER "Include the drawable rendering pathway" OFF)
 option(MLN_USE_UNORDERED_DENSE "Use ankerl dense containers for performance" ON)
 option(MLN_USE_TRACY "Enable Tracy instrumentation" OFF)
 option(MLN_USE_RUST "Use components in Rust" OFF)
+option(MLN_TEXT_SHAPING_HARFBUZZ "Use haffbuzz to shape complex text" ON)
 
 if (MLN_WITH_CLANG_TIDY)
     find_program(CLANG_TIDY_COMMAND NAMES clang-tidy)
@@ -1381,6 +1382,8 @@ if(MLN_DRAWABLE_RENDERER)
     )
 endif()
 
+include(${PROJECT_SOURCE_DIR}/cmake/harfbuzz.cmake)
+
 target_sources(
     mbgl-core PRIVATE
     ${INCLUDE_FILES}
@@ -1484,6 +1487,7 @@ target_link_libraries(
         mbgl-vendor-wagyu
         $<$<BOOL:${MLN_WITH_METAL}>:mbgl-vendor-metal-cpp>
         $<IF:$<BOOL:${MLN_USE_RUST}>,mbgl-rustutils,mbgl-vendor-csscolorparser>
+        ${SHAPE_TARGETS} # from harfbuzz.cmake
     PUBLIC
         Mapbox::Base
         Mapbox::Base::Extras::expected-lite
@@ -1523,6 +1527,7 @@ set(EXPORT_TARGETS
     mbgl-vendor-wagyu
     mbgl-vendor-metal-cpp
     unordered_dense
+    ${SHAPE_TARGETS} # from harfbuzz.cmake
 )
 
 if(MLN_USE_RUST)

MODULE.bazel

@@ -9,6 +9,7 @@ bazel_dep(name = "rules_xcodeproj", version = "2.10.0")
 bazel_dep(name = "aspect_rules_js", version = "2.1.0")
 bazel_dep(name = "rules_nodejs", version = "6.3.2")
 bazel_dep(name = "libuv", version = "1.48.0")
+bazel_dep(name = "rules_foreign_cc", version = "0.9.0")
 
 node = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node", dev_dependency = True)
 node.toolchain(node_version = "20.14.0")

bazel/core.bzl

@@ -545,6 +545,8 @@ MLN_CORE_SOURCE = [
     "src/mbgl/text/shaping.hpp",
     "src/mbgl/text/tagged_string.cpp",
     "src/mbgl/text/tagged_string.hpp",
+    "src/mbgl/text/harfbuzz.cpp",
+    "src/mbgl/text/harfbuzz.hpp",
     "src/mbgl/tile/custom_geometry_tile.cpp",
     "src/mbgl/tile/custom_geometry_tile.hpp",
     "src/mbgl/tile/geojson_tile.cpp",
@@ -1136,3 +1138,10 @@ MLN_DRAWABLES_MTL_HEADERS = [
     "include/mbgl/style/layers/mtl/custom_layer_render_parameters.hpp",
     "include/mbgl/shaders/mtl/widevector.hpp",
 ]
+
+MLN_SHAPING_HARFBUZZ_SRCS = [
+    "src/mbgl/text/freetype.hpp",
+    "src/mbgl/text/freetype.cpp",
+    "src/mbgl/text/harfbuzz_impl.hpp",
+    "src/mbgl/text/harfbuzz_impl.cpp",
+]

cmake/harfbuzz.cmake

@@ -0,0 +1,56 @@
+set(SHAPE_TARGETS, "")
+if (MLN_TEXT_SHAPING_HARFBUZZ) 
+    target_compile_definitions(
+        mbgl-core
+        PRIVATE MLN_TEXT_SHAPING_HARFBUZZ=1
+    )
+    list(APPEND 
+        SRC_FILES
+        ${PROJECT_SOURCE_DIR}/src/mbgl/text/harfbuzz.cpp
+        ${PROJECT_SOURCE_DIR}/src/mbgl/text/harfbuzz.hpp
+        ${PROJECT_SOURCE_DIR}/src/mbgl/text/harfbuzz_impl.cpp
+        ${PROJECT_SOURCE_DIR}/src/mbgl/text/harfbuzz_impl.hpp
+        ${PROJECT_SOURCE_DIR}/src/mbgl/text/freetype.cpp
+        ${PROJECT_SOURCE_DIR}/src/mbgl/text/freetype.hpp
+    )
+
+    set(SHAPE_TARGETS
+        freetype
+        harfbuzz
+    ) 
+
+    set(FT_DISABLE_BROTLI ON CACHE BOOL "freetype option")
+    set(FT_REQUIRE_BROTLI OFF CACHE BOOL "freetype option")
+    add_subdirectory(vendor/freetype)
+    add_subdirectory(vendor/harfbuzz)
+
+    target_compile_definitions(harfbuzz PRIVATE -DHAVE_FREETYPE)
+
+    set_target_properties(
+        freetype
+        PROPERTIES
+            INTERFACE_MAPLIBRE_NAME "freetype"
+            INTERFACE_MAPLIBRE_URL "https://github.com/freetype/freetype"
+            INTERFACE_MAPLIBRE_AUTHOR "David Turner, Robert Wilhelm, Werner Lemberg and FreeType contributors"
+            INTERFACE_MAPLIBRE_LICENSE ${PROJECT_SOURCE_DIR}/vendor/freetype/docs/FTL.TXT
+    )
+
+    set_target_properties(
+        harfbuzz
+        PROPERTIES
+            INTERFACE_MAPLIBRE_NAME "harfbuzz"
+            INTERFACE_MAPLIBRE_URL "https://github.com/harfbuzz/harfbuzz"
+            INTERFACE_MAPLIBRE_AUTHOR ${PROJECT_SOURCE_DIR}/vendor/harfbuzz/AUTHORS
+            INTERFACE_MAPLIBRE_LICENSE ${PROJECT_SOURCE_DIR}/vendor/harfbuzz/COPYING
+    )
+
+    target_include_directories(
+        mbgl-core SYSTEM
+        PUBLIC vendor/freetype/include
+    )
+
+    target_include_directories(
+        mbgl-core SYSTEM
+        PUBLIC vendor/harfbuzz/src
+    )
+endif()

include/mbgl/storage/resource.hpp

@@ -82,6 +82,7 @@ class Resource {
     static Resource glyphs(const std::string& urlTemplate,
                            const FontStack& fontStack,
                            const std::pair<uint16_t, uint16_t>& glyphRange);
+    static Resource fontFace(const std::string& url);
     static Resource spriteImage(const std::string& base, float pixelRatio);
     static Resource spriteJSON(const std::string& base, float pixelRatio);
     static Resource image(const std::string& url);

include/mbgl/text/glyph.hpp

@@ -17,7 +17,40 @@
 
 namespace mbgl {
 
-using GlyphID = char16_t;
+union GlyphID {
+    char32_t hash;
+    struct {
+        char16_t code;
+        GlyphIDType type;
+    } complex;
+
+    GlyphID(int codepoint) {
+        complex.type = FontPBF;
+        complex.code = codepoint;
+    }
+    GlyphID(uint32_t codepoint) {
+        complex.type = FontPBF;
+        complex.code = codepoint;
+    }
+    GlyphID(char16_t codepoint) {
+        complex.type = FontPBF;
+        complex.code = codepoint;
+    }
+
+    GlyphID(char16_t index, GlyphIDType t) {
+        complex.type = t;
+        complex.code = index;
+    }
+
+    operator char16_t() { return complex.code; }
+    operator char32_t() { return hash; }
+    bool operator<(const GlyphID &other) const { return hash < other.hash; }
+    bool operator>(const GlyphID &other) const { return hash > other.hash; }
+
+    bool operator<(const uint16_t &other) const { return hash < other; }
+    bool operator>(const uint16_t &other) const { return hash > other; }
+};
+
 using GlyphIDs = std::set<GlyphID>;
 
 // Note: this only works for the BMP
@@ -31,7 +64,7 @@ struct GlyphMetrics {
     uint32_t advance = 0;
 };
 
-inline bool operator==(const GlyphMetrics& lhs, const GlyphMetrics& rhs) {
+inline bool operator==(const GlyphMetrics &lhs, const GlyphMetrics &rhs) {
     return lhs.width == rhs.width && lhs.height == rhs.height && lhs.left == rhs.left && lhs.top == rhs.top &&
            lhs.advance == rhs.advance;
 }
@@ -113,7 +146,7 @@ class Shaping {
     float right = 0;
     WritingModeType writingMode;
     explicit operator bool() const {
-        return std::any_of(positionedLines.begin(), positionedLines.end(), [](const auto& line) {
+        return std::any_of(positionedLines.begin(), positionedLines.end(), [](const auto &line) {
             return !line.positionedGlyphs.empty();
         });
     }
@@ -129,7 +162,35 @@ enum class WritingModeType : uint8_t {
     Vertical = 1 << 1,
 };
 
-using GlyphDependencies = std::map<FontStack, GlyphIDs>;
+// style defined faces
+struct FontFace {
+    GlyphIDType type; // a unique glyph id
+
+    FontStack fontStack;                               // font stack
+    std::string url;                                   // font file url
+    std::vector<std::pair<uint32_t, uint32_t>> ranges; // unicode ranges
+};
+
+using FontFaces = std::vector<FontFace>;
+
+struct HBShapeRequest {
+    std::u16string str;
+    FontStack fontStack;
+    GlyphIDType type;
+
+    HBShapeRequest(const std::u16string &str_, const FontStack &fontStack_, GlyphIDType type_)
+        : str(str_),
+          fontStack(fontStack_),
+          type(type_) {}
+};
+
+using HBShapeRequests = std::map<FontStack, std::map<GlyphIDType, std::set<std::u16string>>>;
+
+struct GlyphDependencies {
+    std::map<FontStack, GlyphIDs> glyphs;
+    HBShapeRequests shapes;
+};
+
 using GlyphRangeDependencies = std::map<FontStack, std::unordered_set<GlyphRange>>;
 
 } // end namespace mbgl

include/mbgl/text/glyph_range.hpp

@@ -4,10 +4,30 @@
 #include <cstdint>
 #include <unordered_set>
 #include <mbgl/util/hash.hpp>
+#include <mbgl/util/font_stack.hpp>
 
 namespace mbgl {
 
-using GlyphRange = std::pair<uint16_t, uint16_t>;
+enum GlyphIDType : uint16_t {
+    FontPBF = 0x00,
+};
+
+GlyphIDType genNewGlyphIDType(const std::string &url,
+                              const FontStack &fontStack,
+                              const std::vector<std::pair<uint32_t, uint32_t>> &pairs);
+
+class GlyphRange {
+public:
+    uint16_t first = 0;
+    uint16_t second = 0;
+
+    GlyphIDType type = GlyphIDType::FontPBF;
+
+    GlyphRange(uint32_t first_, uint32_t second_, GlyphIDType type_);
+
+    bool operator==(const GlyphRange &other) const;
+    bool operator<(const GlyphRange &other) const;
+};
 
 constexpr uint32_t GLYPHS_PER_GLYPH_RANGE = 256;
 constexpr uint32_t GLYPH_RANGES_PER_FONT_STACK = 256;
@@ -20,7 +40,7 @@ namespace std {
 
 template <>
 struct hash<mbgl::GlyphRange> {
-    std::size_t operator()(const mbgl::GlyphRange& range) const { return mbgl::util::hash(range.first, range.second); }
+    std::size_t operator()(const mbgl::GlyphRange &range) const { return mbgl::util::hash(range.first, range.second); }
 };
 
 } // namespace std

metrics/android-render-test-runner/render-tests/text-font/burmese/metrics.json

@@ -0,0 +1,35 @@
+{
+    "network": [
+        [
+            "probeNetwork - default - end",
+            4,
+            596680
+        ],
+        [
+            "probeNetwork - default - start",
+            0,
+            0
+        ]
+    ],
+    "gfx": [
+        [
+            "probeGFX - default - end",
+            1,
+            13,
+            9,
+            1,
+            [
+                364011,
+                364011
+            ],
+            [
+                90850,
+                90850
+            ],
+            [
+                1211104,
+                1211104
+            ]
+        ]
+    ]
+}