Introduce new and more flexible API grapheme_line_segmenter, replacin…

…g scan API

Signed-off-by: Christian Parpart <christian@parpart.family>

.gitignore

@@ -4,6 +4,7 @@
 /_ucd/
 /.clangd/
 /compile_commands.json
+/.vs/
 /.vscode/
 /sandbox/
 /target/

CMakeLists.txt

@@ -32,7 +32,8 @@ if(("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" MAT
     endif()
 elseif(DEFINED MSVC)
     add_definitions(-DNOMINMAX)
-    add_compile_options(/utf-8)
+    add_compile_options(/nologo)
+    set(CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} /nologo")
 endif()
 
 include(EnableCcache)
@@ -47,7 +48,7 @@ option(LIBUNICODE_BENCHMARK "libunicode: Enables building of benchmark for libun
 option(LIBUNICODE_TOOLS "libunicode: Builds CLI tools [default: ${MASTER_PROJECT}]" ${MASTER_PROJECT})
 option(LIBUNICODE_BUILD_STATIC "libunicode: provide static library instead of dynamic [default: ${LIBUNICODE_BUILD_STATIC_DEFAULT}]" ${LIBUNICODE_BUILD_STATIC_DEFAULT})
 option(LIBUNICODE_USE_INTRINSICS "libunicode: Use SIMD extenstion during text read [default: ON]" ON)
-option(LIBUNICODE_USE_STD_SIMD "libunicode: Use std::simd as SIMD extenstion during text read (takes precedence over own intrinsics) [default: ON]" ${LIBUNICODE_USE_INTRINSICS})
+option(LIBUNICODE_USE_STD_SIMD "libunicode: Use std::simd as SIMD extenstion during text read (takes precedence over own intrinsics) [default: ON]" ON)
 option(LIBUNICODE_TABLEGEN_FASTBUILD "libunicode: Use fast table generation (takes more memory in final tables) [default: OFF]" OFF)
 
 set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Enable testing of the benchmark library." FORCE)

cmake/presets/common.json

@@ -1,7 +1,7 @@
 {
     "version": 6,
     "configurePresets": [
-        { "name": "debug", "hidden": true, "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", "LIBUNICODE_TABLEGEN_FASTBUILD": "ON" } },
+        { "name": "debug", "hidden": true, "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", "LIBUNICODE_TABLEGEN_FASTBUILD": "ON", "LIBUNICODE_TRACE": "ON" } },
         { "name": "release", "hidden": true, "cacheVariables": { "CMAKE_BUILD_TYPE": "RelWithDebInfo" } },
         { "name": "arch-native", "hidden": true, "cacheVariables": { "CMAKE_CXX_FLAGS": "-march=native" } },
         { "name": "clang", "hidden": true, "cacheVariables": { "CMAKE_CXX_COMPILER": "clang++" } },

cmake/presets/os-windows.json

@@ -16,8 +16,8 @@
             "cacheVariables": {
                 "VCPKG_TARGET_TRIPLET": "x64-windows",
                 "CMAKE_INSTALL_PREFIX": "${sourceDir}/out/install/${presetName}",
-                "CMAKE_VERBOSE_MAKEFILE": "ON",
-                "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/../vcpkg/scripts/buildsystems/vcpkg.cmake"
+                "CMAKE_TOOLCHAIN_FILE": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake",
+                "CMAKE_VERBOSE_MAKEFILE": "ON"
             }
         },
         { "name": "windows-cl-debug", "inherits": ["windows-common", "debug"], "displayName": "Windows (MSVC) Debug", "description": "Using MSVC compiler (64-bit)" },

src/libunicode/CMakeLists.txt

@@ -1,5 +1,7 @@
 include(GNUInstallDirs)
 
+option(LIBUNICODE_TRACE "Enable trace logging" OFF)
+
 function(ExtractZipArchive ZIP_FILE OUTPUT_DIR)
     if(CMAKE_VERSION VERSION_LESS 3.18)
         # Use the older method for versions prior to CMake 3.18
@@ -84,6 +86,9 @@ add_custom_command(
 
 add_library(unicode_loader ${LIBUNICODE_LIB_MODE} codepoint_properties_loader.h codepoint_properties_loader.cpp)
 add_library(unicode::loader ALIAS unicode_loader)
+if(DEFINED MSVC)
+    target_compile_options(unicode_loader PUBLIC /EHsc) # We currently `throw` in the loader, so we need this.
+endif()
 if(LIBUNICODE_TABLEGEN_FASTBUILD)
     target_compile_definitions(unicode_loader PRIVATE LIBUNICODE_TABLEGEN_FASTBUILD)
 endif()
@@ -102,7 +107,6 @@ add_library(unicode ${LIBUNICODE_LIB_MODE}
     codepoint_properties.cpp
     emoji_segmenter.cpp
     grapheme_segmenter.cpp
-    scan.cpp
     script_segmenter.cpp
     utf8.cpp
     width.cpp
@@ -114,22 +118,22 @@ add_library(unicode ${LIBUNICODE_LIB_MODE}
 )
 
 if(LIBUNICODE_USE_STD_SIMD)
-    target_compile_definitions(unicode PRIVATE LIBUNICODE_USE_STD_SIMD)
+    target_compile_definitions(unicode PUBLIC LIBUNICODE_USE_STD_SIMD)
 endif()
 if(LIBUNICODE_USE_INTRINSICS)
-    target_compile_definitions(unicode PRIVATE USE_INTRINSICS)
+    target_compile_definitions(unicode PUBLIC LIBUNICODE_USE_INTRINSICS)
 endif()
 
 set(public_headers
     capi.h
     codepoint_properties.h
     convert.h
     emoji_segmenter.h
+    grapheme_line_segmenter.h
     grapheme_segmenter.h
     intrinsics.h
     multistage_table_view.h
     run_segmenter.h
-    scan.h
     script_segmenter.h
     support.h
     utf8.h
@@ -150,6 +154,10 @@ set_target_properties(unicode PROPERTIES
     SOVERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}"
 )
 
+if(LIBUNICODE_TRACE)
+    target_compile_definitions(unicode PUBLIC LIBUNICODE_TRACE)
+endif()
+
 add_library(unicode::unicode ALIAS unicode)
 add_library(unicode::core ALIAS unicode)
 target_include_directories(unicode PUBLIC $<BUILD_INTERFACE:${${PROJECT_NAME}_SOURCE_DIR}/src>
@@ -161,7 +169,6 @@ add_executable(unicode_tablegen tablegen.cpp)
 set_target_properties(unicode_tablegen PROPERTIES CMAKE_BUILD_TYPE Release)
 target_link_libraries(unicode_tablegen PRIVATE unicode::loader)
 
-
 # {{{ installation
 set(LIBUNICODE_CMAKE_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/libunicode" CACHE PATH "Installation directory for cmake files, a relative path that will be joined with ${CMAKE_INSTALL_PREFIX} or an absolute path.")
 set(LIBUNICODE_INSTALL_CMAKE_FILES ${MASTER_PROJECT} CACHE BOOL "Decides whether or not to install CMake config and -version files.")
@@ -220,35 +227,35 @@ if(LIBUNICODE_TESTING)
         capi_test.cpp
         convert_test.cpp
         emoji_segmenter_test.cpp
+        grapheme_line_segmenter_test.cpp
         grapheme_segmenter_test.cpp
         run_segmenter_test.cpp
-        scan_test.cpp
         script_segmenter_test.cpp
         test_main.cpp
-        unicode_test.cpp
         utf8_grapheme_segmenter_test.cpp
         utf8_test.cpp
         width_test.cpp
         word_segmenter_test.cpp
     )
 
+    if(DEFINED MSVC)
+        target_compile_options(unicode_test PRIVATE /utf-8)
+    endif()
+
     if(NOT Catch2_FOUND)
         # supress conversion warnings for Catch2
         # https://github.com/catchorg/Catch2/issues/2583
         # https://github.com/SFML/SFML/blob/e45628e2ebc5843baa3739781276fa85a54d4653/test/CMakeLists.txt#L18-L22
         set_target_properties(Catch2 PROPERTIES COMPILE_OPTIONS "" EXPORT_COMPILE_COMMANDS OFF)
-        set_target_properties(Catch2WithMain PROPERTIES EXPORT_COMPILE_COMMANDS OFF)
         get_target_property(CATCH2_INCLUDE_DIRS Catch2 INTERFACE_INCLUDE_DIRECTORIES)
         target_include_directories(Catch2 SYSTEM INTERFACE ${CATCH2_INCLUDE_DIRS})
     endif()
 
-    target_link_libraries(unicode_test unicode Catch2::Catch2WithMain fmt::fmt-header-only)
+    target_link_libraries(unicode_test unicode Catch2::Catch2 fmt::fmt-header-only)
     add_test(unicode_test unicode_test)
 endif()
 # }}}
 
-
-
 # {{{ unicode_test
 if(LIBUNICODE_BENCHMARK)
     if(NOT benchmark_FOUND)

src/libunicode/benchmark.cpp

@@ -1,5 +1,5 @@
 #include <libunicode/convert.h>
-#include <libunicode/scan.h>
+#include <libunicode/grapheme_line_segmenter.h>
 #include <libunicode/utf8.h>
 
 #include <string_view>
@@ -14,7 +14,7 @@ static void benchmarkWithLength(benchmark::State& benchmarkState)
     auto TestText = std::string(L, 'a') + "\u00A9";
     for (auto _: benchmarkState)
     {
-        benchmark::DoNotOptimize(unicode::detail::scan_for_text_ascii(TestText, L + 10));
+        benchmark::DoNotOptimize(unicode::detail::process_only_ascii(std::string_view(TestText).substr(0, L + 10)));
     }
 }
 
@@ -24,7 +24,9 @@ static void benchmarkWithOffset(benchmark::State& benchmarkState)
     auto TestText = std::string(L, 'a') + "\U0001F600" + std::string(1000, 'a');
     for (auto _: benchmarkState)
     {
-        benchmark::DoNotOptimize(unicode::detail::scan_for_text_ascii(TestText, L + 10));
+        auto state = unicode::detail::unicode_process_state {};
+        auto eventHandler = unicode::detail::EventHandler{};
+        benchmark::DoNotOptimize(unicode::detail::process_only_complex_unicode(eventHandler, state, TestText, L + 10));
     }
 }