feat(C++): The use of SIMD accelerated to implement and optimize utf1…

…6 utf8 (#1732)

<!--
**Thanks for contributing to Fury.**

**If this is your first time opening a PR on fury, you can refer to
[CONTRIBUTING.md](https://github.com/apache/fury/blob/main/CONTRIBUTING.md).**

Contribution Checklist

- The **Apache Fury (incubating)** community has restrictions on the
naming of pr titles. You can also find instructions in
[CONTRIBUTING.md](https://github.com/apache/fury/blob/main/CONTRIBUTING.md).

- Fury has a strong focus on performance. If the PR you submit will have
an impact on performance, please benchmark it first and provide the
benchmark result here.
-->

## What does this PR do?
Use SIMD acceleration to implement and optimize utf16 to utf8.
Adapter x86, arm and  risc-v
Add 8 tests verify the function
<img width="261" alt="fury_cpp_simd_utf_1"
src="https://github.com/user-attachments/assets/029fe6ea-b4be-4e26-85d2-3c5e02e64899">
And efficiency has improved
<img width="401" alt="fury_cpp_simd_utf_2"
src="https://github.com/user-attachments/assets/6e86c125-f5a2-46df-b3bd-3d12496e9238">
done.


## Related issues
Closes #1546 
<!--
Is there any related issue? Please attach here.

- #xxxx0
- #xxxx1
- #xxxx2
-->


## Does this PR introduce any user-facing change?

<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/fury/issues/new/choose) describing the
need to do so and update the document if necessary.
-->

- [x] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?

cpp/fury/util/string_util.cc

@@ -27,8 +27,37 @@
 #include <riscv_vector.h>
 #endif
 
+#include <chrono>
+#include <string>
+
 namespace fury {
 
+// Swap bytes to convert from big endian to little endian
+inline uint16_t swapBytes(uint16_t value) {
+ return (value >> 8) | (value << 8);
+}
+
+inline void utf16ToUtf8(uint16_t code_unit, char *&output) {
+ if (code_unit < 0x80) {
+ *output++ = static_cast<char>(code_unit);
+ } else if (code_unit < 0x800) {
+ *output++ = static_cast<char>(0xC0 | (code_unit >> 6));
+ *output++ = static_cast<char>(0x80 | (code_unit & 0x3F));
+ } else {
+ *output++ = static_cast<char>(0xE0 | (code_unit >> 12));
+ *output++ = static_cast<char>(0x80 | ((code_unit >> 6) & 0x3F));
+ *output++ = static_cast<char>(0x80 | (code_unit & 0x3F));
+ }
+}
+
+inline void utf16SurrogatePairToUtf8(uint16_t high, uint16_t low, char *&utf8) {
+ uint32_t code_point = 0x10000 + ((high - 0xD800) << 10) + (low - 0xDC00);
+ *utf8++ = static_cast<char>((code_point >> 18) | 0xF0);
+ *utf8++ = static_cast<char>(((code_point >> 12) & 0x3F) | 0x80);
+ *utf8++ = static_cast<char>(((code_point >> 6) & 0x3F) | 0x80);
+ *utf8++ = static_cast<char>((code_point & 0x3F) | 0x80);
+}
+
 #if defined(__x86_64__) || defined(_M_X64)
 
 bool isLatin(const std::string &str) {
@@ -55,6 +84,90 @@ bool isLatin(const std::string &str) {
  return true;
 }
 
+std::string utf16ToUtf8(const std::u16string &utf16, bool is_little_endian) {
+ std::string utf8;
+ utf8.reserve(utf16.size() *
+ 3); // Reserve enough space to avoid frequent reallocations
+
+ const __m256i limit1 = _mm256_set1_epi16(0x80);
+ const __m256i limit2 = _mm256_set1_epi16(0x800);
+ const __m256i surrogate_high_start = _mm256_set1_epi16(0xD800);
+ const __m256i surrogate_high_end = _mm256_set1_epi16(0xDBFF);
+ const __m256i surrogate_low_start = _mm256_set1_epi16(0xDC00);
+ const __m256i surrogate_low_end = _mm256_set1_epi16(0xDFFF);
+
+ char buffer[64]; // Buffer to hold temporary UTF-8 bytes
+ char *output = buffer;
+
+ size_t i = 0;
+ size_t n = utf16.size();
+
+ while (i + 16 <= n) {
+ __m256i in =
+ _mm256_loadu_si256(reinterpret_cast<const __m256i *>(utf16.data() + i));
+
+ if (!is_little_endian) {
+ in = _mm256_or_si256(
+ _mm256_slli_epi16(in, 8),
+ _mm256_srli_epi16(in, 8)); // Swap bytes for big-endian
+ }
+
+ __m256i mask1 = _mm256_cmpgt_epi16(in, limit1);
+ __m256i mask2 = _mm256_cmpgt_epi16(in, limit2);
+ __m256i high_surrogate_mask =
+ _mm256_and_si256(_mm256_cmpgt_epi16(in, surrogate_high_start),
+ _mm256_cmpgt_epi16(in, surrogate_high_end));
+ __m256i low_surrogate_mask =
+ _mm256_and_si256(_mm256_cmpgt_epi16(in, surrogate_low_start),
+ _mm256_cmpgt_epi16(in, surrogate_low_end));
+
+ if (_mm256_testz_si256(mask1, mask1)) {
+ // All values < 0x80, 1 byte per character
+ for (int j = 0; j < 16; ++j) {
+ *output++ = static_cast<char>(utf16[i + j]);
+ }
+ } else if (_mm256_testz_si256(mask2, mask2)) {
+ // All values < 0x800, 2 bytes per character
+ for (int j = 0; j < 16; ++j) {
+ utf16ToUtf8(utf16[i + j], output);
+ }
+ } else {
+ // Mix of 1, 2, and 3 byte characters
+ for (int j = 0; j < 16; ++j) {
+ if (_mm256_testz_si256(high_surrogate_mask, high_surrogate_mask) &&
+ j + 1 < 16 &&
+ !_mm256_testz_si256(low_surrogate_mask, low_surrogate_mask)) {
+ // Surrogate pair
+ utf16SurrogatePairToUtf8(utf16[i + j], utf16[i + j + 1], output);
+ ++j;
+ } else {
+ utf16ToUtf8(utf16[i + j], output);
+ }
+ }
+ }
+
+ utf8.append(buffer, output - buffer);
+ output = buffer; // Reset output buffer pointer
+ i += 16;
+ }
+
+ // Handle remaining characters
+ while (i < n) {
+ if (i + 1 < n && utf16[i] >= 0xD800 && utf16[i] <= 0xDBFF &&
+ utf16[i + 1] >= 0xDC00 && utf16[i + 1] <= 0xDFFF) {
+ // Surrogate pair
+ utf16SurrogatePairToUtf8(utf16[i], utf16[i + 1], output);
+ ++i;
+ } else {
+ utf16ToUtf8(utf16[i], output);
+ }
+ ++i;
+ }
+ utf8.append(buffer, output - buffer);
+
+ return utf8;
+}
+
 #elif defined(__ARM_NEON) || defined(__ARM_NEON__)
 
 bool isLatin(const std::string &str) {
@@ -80,18 +193,89 @@ bool isLatin(const std::string &str) {
  return true;
 }
 
+std::string utf16ToUtf8(const std::u16string &utf16, bool is_little_endian) {
+ std::string utf8;
+ utf8.reserve(utf16.size() * 3);
+
+ uint16x8_t limit1 = vdupq_n_u16(0x80);
+ uint16x8_t limit2 = vdupq_n_u16(0x800);
+ uint16x8_t surrogate_high_start = vdupq_n_u16(0xD800);
+ uint16x8_t surrogate_high_end = vdupq_n_u16(0xDBFF);
+ uint16x8_t surrogate_low_start = vdupq_n_u16(0xDC00);
+ uint16x8_t surrogate_low_end = vdupq_n_u16(0xDFFF);
+
+ char buffer[64];
+ char *output = buffer;
+ size_t i = 0;
+ size_t n = utf16.size();
+
+ while (i + 8 <= n) {
+ uint16x8_t in =
+ vld1q_u16(reinterpret_cast<const uint16_t *>(utf16.data() + i));
+ if (!is_little_endian) {
+ in = vorrq_u16(vshlq_n_u16(in, 8),
+ vshrq_n_u16(in, 8)); // Swap bytes for big-endian
+ }
+
+ uint16x8_t mask1 = vcgtq_u16(in, limit1);
+ uint16x8_t mask2 = vcgtq_u16(in, limit2);
+ uint16x8_t high_surrogate_mask = vandq_u16(
+ vcgtq_u16(in, surrogate_high_start), vcltq_u16(in, surrogate_high_end));
+ uint16x8_t low_surrogate_mask = vandq_u16(
+ vcgtq_u16(in, surrogate_low_start), vcltq_u16(in, surrogate_low_end));
+
+ if (vmaxvq_u16(mask1) == 0) {
+ for (int j = 0; j < 8; ++j) {
+ *output++ = static_cast<char>(utf16[i + j]);
+ }
+ } else if (vmaxvq_u16(mask2) == 0) {
+ for (int j = 0; j < 8; ++j) {
+ utf16ToUtf8(utf16[i + j], output);
+ }
+ } else {
+ for (int j = 0; j < 8; ++j) {
+ if (vmaxvq_u16(high_surrogate_mask) == 0 && j + 1 < 8 &&
+ vmaxvq_u16(low_surrogate_mask) != 0) {
+ utf16SurrogatePairToUtf8(utf16[i + j], utf16[i + j + 1], output);
+ ++j;
+ } else {
+ utf16ToUtf8(utf16[i + j], output);
+ }
+ }
+ }
+
+ utf8.append(buffer, output - buffer);
+ output = buffer;
+ i += 8;
+ }
+
+ while (i < n) {
+ if (i + 1 < n && utf16[i] >= 0xD800 && utf16[i] <= 0xDBFF &&
+ utf16[i + 1] >= 0xDC00 && utf16[i + 1] <= 0xDFFF) {
+ utf16SurrogatePairToUtf8(utf16[i], utf16[i + 1], output);
+ ++i;
+ } else {
+ utf16ToUtf8(utf16[i], output);
+ }
+ ++i;
+ }
+ utf8.append(buffer, output - buffer);
+
+ return utf8;
+}
+
 #elif defined(__riscv) && __riscv_vector
 
 bool isLatin(const std::string &str) {
  const char *data = str.data();
  size_t len = str.size();
 
  size_t i = 0;
+ auto latin_mask = vmv_v_x_u8m1(0x80, 16);
  for (; i + 16 <= len; i += 16) {
  auto chars = vle8_v_u8m1(reinterpret_cast<const uint8_t *>(data + i), 16);
- auto mask = vmv_v_x_u8m1(0x80, 16);
- auto result = vand_vv_u8m1(chars, mask, 16);
- if (vmax_v_u8m1(result, 16) != 0) {
+ auto result = vand_vv_u8m1(chars, latin_mask, 16);
+ if (vfirst_m_b8(vmsne_vx_u8m1_b8(result, 0, 16))) {
  return false;
  }
  }
@@ -105,6 +289,82 @@ bool isLatin(const std::string &str) {
  return true;
 }
 
+std::string utf16ToUtf8(const std::u16string &utf16, bool is_little_endian) {
+ std::string utf8;
+ utf8.reserve(utf16.size() * 3);
+
+ auto limit1 = vmv_v_x_u16m1(0x80, 8);
+ auto limit2 = vmv_v_x_u16m1(0x800, 8);
+ auto surrogate_high_start = vmv_v_x_u16m1(0xD800, 8);
+ auto surrogate_high_end = vmv_v_x_u16m1(0xDBFF, 8);
+ auto surrogate_low_start = vmv_v_x_u16m1(0xDC00, 8);
+ auto surrogate_low_end = vmv_v_x_u16m1(0xDFFF, 8);
+
+ char buffer[48];
+ char *output = buffer;
+ size_t i = 0;
+ size_t n = utf16.size();
+
+ while (i + 8 <= n) {
+ auto in =
+ vle16_v_u16m1(reinterpret_cast<const uint16_t *>(utf16.data() + i), 8);
+ if (!is_little_endian) {
+ in = vor_vv_u16m1(vsrl_vx_u16m1(in, 8, 8), vsll_vx_u16m1(in, 8, 8), 8);
+ }
+
+ auto mask1 = vmsgt_vx_u16m1(in, 0x80, 8);
+ auto mask2 = vmsgt_vx_u16m1(in, 0x800, 8);
+ auto high_surrogate_mask = vmand_vv_u16m1(vmsgt_vx_u16m1(in, 0xD800, 8),
+ vmslt_vx_u16m1(in, 0xDBFF, 8), 8);
+ auto low_surrogate_mask = vmand_vv_u16m1(vmsgt_vx_u16m1(in, 0xDC00, 8),
+ vmslt_vx_u16m1(in, 0xDFFF, 8), 8);
+
+ if (vmslt_vx_u16m1(mask1, 0, 8)) {
+ for (int j = 0; j < 8; ++j) {
+ *output++ = static_cast<char>(vget_vx_u16m1(in, j));
+ }
+ } else if (vmslt_vx_u16m1(mask2, 0, 8)) {
+ for (int j = 0; j < 8; ++j) {
+ utf16ToUtf8(vget_vx_u16m1(in, j), output);
+ }
+ } else {
+ for (int j = 0; j < 8; ++j) {
+ if (vfirst_m_b8(
+ vmand_vv_b8(high_surrogate_mask,
+ vmsne_vx_u8m1_b8(vmv_v_x_u8m1(0, 8), 0, 8))) &&
+ j + 1 < 8 &&
+ vfirst_m_b8(
+ vmand_vv_b8(low_surrogate_mask,
+ vmsne_vx_u8m1_b8(vmv_v_x_u8m1(0, 8), 0, 8)))) {
+ utf16SurrogatePairToUtf8(vget_vx_u16m1(in, j),
+ vget_vx_u16m1(in, j + 1), output);
+ ++j;
+ } else {
+ utf16ToUtf8(vget_vx_u16m1(in, j), output);
+ }
+ }
+ }
+
+ utf8.append(buffer, output - buffer);
+ output = buffer;
+ i += 8;
+ }
+
+ while (i < n) {
+ if (i + 1 < n && utf16[i] >= 0xD800 && utf16[i] <= 0xDBFF &&
+ utf16[i + 1] >= 0xDC00 && utf16[i + 1] <= 0xDFFF) {
+ utf16SurrogatePairToUtf8(utf16[i], utf16[i + 1], output);
+ ++i;
+ } else {
+ utf16ToUtf8(utf16[i], output);
+ }
+ ++i;
+ }
+ utf8.append(buffer, output - buffer);
+
+ return utf8;
+}
+
 #else
 
 bool isLatin(const std::string &str) {
@@ -116,6 +376,44 @@ bool isLatin(const std::string &str) {
  return true;
 }
 
+// Fallback implementation without SIMD acceleration
+std::string utf16ToUtf8(const std::u16string &utf16, bool is_little_endian) {
+ std::string utf8;
+ utf8.reserve(utf16.size() *
+ 3); // Reserve enough space to avoid frequent reallocations
+
+ size_t i = 0;
+ size_t n = utf16.size();
+ char buffer[4]; // Buffer to hold temporary UTF-8 bytes
+ char *output = buffer;
+
+ while (i < n) {
+ uint16_t code_unit = utf16[i];
+ if (!is_little_endian) {
+ code_unit = swapBytes(code_unit);
+ }
+ if (i + 1 < n && code_unit >= 0xD800 && code_unit <= 0xDBFF &&
+ utf16[i + 1] >= 0xDC00 && utf16[i + 1] <= 0xDFFF) {
+ // Surrogate pair
+ uint16_t high = code_unit;
+ uint16_t low = utf16[i + 1];
+ if (!is_little_endian) {
+ low = swapBytes(low);
+ }
+ utf16SurrogatePairToUtf8(high, low, output);
+ utf8.append(buffer, output - buffer);
+ output = buffer;
+ ++i;
+ } else {
+ utf16ToUtf8(code_unit, output);
+ utf8.append(buffer, output - buffer);
+ output = buffer;
+ }
+ ++i;
+ }
+ return utf8;
+}
+
 #endif
 
 } // namespace fury

cpp/fury/util/string_util.h

@@ -25,4 +25,6 @@ namespace fury {
 
 bool isLatin(const std::string &str);
 
+std::string utf16ToUtf8(const std::u16string &utf16, bool is_little_endian);
+
 } // namespace fury