diff --git a/.gitignore b/.gitignore index fed98a78..5373c586 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ cmake-build-debug/ build/ *.pb.h *.pb.cc + diff --git a/LICENSE b/LICENSE index 52ce1092..828d1e17 100644 --- a/LICENSE +++ b/LICENSE @@ -454,6 +454,56 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI As an exception, if, as a result of your compiling your source code, portions of this Software are embedded into a machine-executable object form of such source code, you may redistribute such embedded portions in such object form without including the above copyright and permission notices. +Open Source Software Licensed under the MIT: +-------------------------------------------------------------------- +1. murmurhash3 +Copyright (C) 2011 - present, Austin Appleby + + +Terms of the MIT: +-------------------------------------------------------------------- +MurmurHash3 was written by Austin Appleby, and is placed in the public domain. The author hereby disclaims copyright to this source code. + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +Open Source Software Licensed under the MIT: +-------------------------------------------------------------------- +1. city +Copyright (c) 2011 Google, Inc. + + +Terms of the MIT: +-------------------------------------------------------------------- +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN HE SOFTWARE. + +CityHash, by Geoff Pike and Jyrki Alakuijala + + +Open Source Software Licensed under the MIT: +-------------------------------------------------------------------- +1. md5 +Copyright (C) 1991-2, RSA Data Security, Inc + + +Terms of the MIT: +-------------------------------------------------------------------- +License to copy and use this software is granted provided that it is identified as the "RSA Data Security, Inc. MD5 Message-Digest Algorithm" in all material mentioning or referencing this software +or this function. License is also granted to make and use derivative works provided that such works are identified as "derived from the RSA Data Security, Inc. MD5 Message-Digest Algorithm" in all material mentioning or referencing the derived work. + +RSA Data Security, Inc. makes no representations concerning either the merchantability of this software or the suitability of this software for any particular purpose. It is provided "as is" +without express or implied warranty of any kind. + +These notices must be retained in any copies of any part of this +documentation and/or software. + + Open Source Software Licensed under the OpenSSL License and the original SSLeay License: -------------------------------------------------------------------- 1. openssl diff --git a/docs/zh/hash_loadbalance.md b/docs/zh/hash_loadbalance.md new file mode 100644 index 00000000..eabda616 --- /dev/null +++ b/docs/zh/hash_loadbalance.md @@ -0,0 +1,81 @@ +# 哈希负载均衡插件使用 + +## 使用哈希负载均衡插件 + +要使用哈希负载均衡插件，需在yaml配置文件client:service:load_balance_name 下设置哈希负载均衡插件名字，并在plugins:loadbalance下配置对应插件的相关配置。 + +以下是一个在yaml配置文件使用哈希负载均衡插件的例子。 + +``` +client: + service: + - name: trpc.test.helloworld.Greeter + target: 127.0.0.1:11111,127.0.0.1:22222,127.0.0.1:33333 # Fullfill ip:port list here when use `direct` selector.(such as 23.9.0.1:90,34.5.6.7:90) + protocol: trpc # Application layer protocol, eg: trpc/http/... + network: tcp # Network type, Support two types: tcp/udp + selector_name: direct # Selector plugin, default `direct`, it is used when you want to access via ip:port + load_balance_name: consistent_hash + +plugins: + loadbalance: + consistent_hash: + hash_nodes: 20 + hash_args: [0] + hash_func: murmur3 +``` + +## 默认负载均衡插件 + +若没有在client端设置load_balance_name, 默认采用轮询负载均衡插件（trpc_polling_load_balance)，轮询负载均衡插件不需要在plugins下进行配置。 + +## 哈希负载均衡插件 + +如果用户在client端设置了hash值，将采用用户提供的hash值来进行路由选择。否则将使用插件的配置来生成hash值（生成hash值的函数见 哈希函数使用 章节） + +### 一致性哈希负载均衡插件（consistent_hash) + +以下配置值为默认值，若没对插件进行配置，将采用下面的默认值 + +``` +plugins: + loadbalance: + consistent_hash: + hash_nodes: 160 #consistent hash中每个实际节点对应的虚拟节点数量 + hash_args: [0] #支持0-5选项，分别对应selectInfo中的信息.0：caller name 1: client ip 2：client port 3:info.name 4: callee name 5: info.loadbalance name + hash_func: murmur3 #支持murmur3，city，md5，bkdr，fnv1a +``` + +### 取模哈希负载均衡插件（modulo_hash) + +以下配置值为默认值，若没对插件进行配置，将采用下面的默认值 + +``` +plugins: + loadbalance: + modulo_hash: + hash_args: [0] + hash_func: murmur3 +``` + +### 哈希函数使用 + +在哈希负载均衡插件中使用的哈希函数的定义在文件/trpc/naming/common/util/hash/hash_func.h + +文件提供的哈希函数如下： + +``` +//input为输入的键，hash_func为选择的hash函数，支持“murmur3”，“city”，“md5”，“bkdr”，“fnv1a” +//返回64位哈希值 +std::uint64_t Hash(const std::string& input, const std::string& hash_func); + +//input为输入的键，hash_func为选择的hash函数，支持“murmur3”，“city”，“md5”，“bkdr”，“fnv1a”，num为模数 +//返回64位取模后的哈希值 +std::uint64_t Hash(const std::string& input, const std::string& hash_func, uint64_t num); + +//input为输入的键，hash_func为选择的hash函数，支持HashFuncName::MD5,HashFuncName::BKDR,HashFuncName::CITY,HashFuncName::BKDR,HashFuncName::MURMUR3,HashFuncName::FNV1A +std::uint64_t Hash(const std::string& input, const HashFuncName& hash_func); + +//取模后的哈希值 +std::uint64_t Hash(const std::string& input, const HashFuncName& hash_func,uint64_t num); + +``` diff --git a/trpc/common/BUILD b/trpc/common/BUILD index 647cd6ca..20e81d7f 100644 --- a/trpc/common/BUILD +++ b/trpc/common/BUILD @@ -137,7 +137,7 @@ cc_library( "//trpc/util/log/default:default_log", "//trpc/util:net_util", "//trpc/overload_control:trpc_overload_control", - "//trpc/naming/common/util/loadbalance:trpc_load_balance" + "//trpc/naming/common/util/loadbalance:trpc_load_balance", ] + select({ "//trpc:trpc_include_rpcz": [ "//trpc/rpcz:collector", diff --git a/trpc/common/config/BUILD b/trpc/common/config/BUILD index 53fa30ac..56d85380 100644 --- a/trpc/common/config/BUILD +++ b/trpc/common/config/BUILD @@ -316,6 +316,24 @@ cc_test( ], ) +cc_library( + name = "loadbalance_naming_conf", + srcs = ["loadbalance_naming_conf.cc"], + hdrs = ["loadbalance_naming_conf.h"], + deps = [ + "//trpc/util/log:logging", + "@com_github_jbeder_yaml_cpp//:yaml-cpp", + ], +) + +cc_library( + name = "loadbalance_naming_conf_parser", + hdrs = ["loadbalance_naming_conf_parser.h"], + deps = [ + ":loadbalance_naming_conf", + ], +) + cc_library( name = "local_file_provider_conf", srcs = ["local_file_provider_conf.cc"], diff --git a/trpc/common/config/loadbalance_naming_conf.cc b/trpc/common/config/loadbalance_naming_conf.cc new file mode 100644 index 00000000..e226e54a --- /dev/null +++ b/trpc/common/config/loadbalance_naming_conf.cc @@ -0,0 +1,30 @@ +// +// +// Tencent is pleased to support the open source community by making tRPC available. +// +// Copyright (C) 2024 THL A29 Limited, a Tencent company. +// All rights reserved. +// +// If you have downloaded a copy of the tRPC source code from Tencent, +// please note that tRPC source code is licensed under the Apache 2.0 License, +// A copy of the Apache 2.0 License is included in this file. +// +// + +#include "trpc/common/config/loadbalance_naming_conf.h" + +#include "trpc/util/log/logging.h" + +namespace trpc::naming { + +void LoadBalanceConfig::Display() const { + TRPC_FMT_DEBUG("-----LoadBalanceSelectorConfig begin-------"); + + TRPC_FMT_DEBUG("hash_nodes:{}", hash_nodes); + TRPC_FMT_DEBUG("hash_args size:{}", hash_args.size()); + TRPC_FMT_DEBUG("hash_func:{}", hash_func); + + TRPC_FMT_DEBUG("--------------------------------------"); +} + +} // namespace trpc::naming \ No newline at end of file diff --git a/trpc/common/config/loadbalance_naming_conf.h b/trpc/common/config/loadbalance_naming_conf.h new file mode 100644 index 00000000..dde0f7d0 --- /dev/null +++ b/trpc/common/config/loadbalance_naming_conf.h @@ -0,0 +1,36 @@ +// +// +// Tencent is pleased to support the open source community by making tRPC available. +// +// Copyright (C) 2024 THL A29 Limited, a Tencent company. +// All rights reserved. +// +// If you have downloaded a copy of the tRPC source code from Tencent, +// please note that tRPC source code is licensed under the Apache 2.0 License, +// A copy of the Apache 2.0 License is included in this file. +// +// + +#pragma once + +#include +#include +#include + +namespace trpc::naming { + +/// @brief loadbalance plugin configuration +struct LoadBalanceConfig { + /// @brief hash node number when consistent hash load balance algorithm is hash + uint32_t hash_nodes{160}; + /// @brief hash content when load balance algorithm is hash,corresponding to SelectInfo + /// 0：caller name 1: client ip 2：client port 3:info.name 4: callee name 5: info.loadbalance name + std::vector hash_args{0}; + /// @brief hash function when load balance algorithm is hash + std::string hash_func{"murmur3"}; + + /// @brief Print out the logger configuration. + void Display() const; +}; + +} // namespace trpc::naming \ No newline at end of file diff --git a/trpc/common/config/loadbalance_naming_conf_parser.h b/trpc/common/config/loadbalance_naming_conf_parser.h new file mode 100644 index 00000000..7dc437d3 --- /dev/null +++ b/trpc/common/config/loadbalance_naming_conf_parser.h @@ -0,0 +1,44 @@ +// +// +// Tencent is pleased to support the open source community by making tRPC available. +// +// Copyright (C) 2024 THL A29 Limited, a Tencent company. +// All rights reserved. +// +// If you have downloaded a copy of the tRPC source code from Tencent, +// please note that tRPC source code is licensed under the Apache 2.0 License, +// A copy of the Apache 2.0 License is included in this file. +// +// +#pragma once + +#include "yaml-cpp/yaml.h" + +#include "trpc/common/config/loadbalance_naming_conf.h" + +namespace YAML { + +template <> +struct convert { + static YAML::Node encode(const trpc::naming::LoadBalanceConfig& config) { + YAML::Node node; + node["hash_nodes"] = config.hash_nodes; + node["hash_args"] = config.hash_args; + node["hash_func"] = config.hash_func; + return node; + } + + static bool decode(const YAML::Node& node, trpc::naming::LoadBalanceConfig& config) { + if (node["hash_nodes"]) { + config.hash_nodes = node["hash_nodes"].as(); + } + if (node["hash_args"]) { + config.hash_args = node["hash_args"].as>(); + } + if (node["hash_func"]) { + config.hash_func = node["hash_func"].as(); + } + return true; + } +}; +} // namespace YAML \ No newline at end of file diff --git a/trpc/common/trpc_plugin.cc b/trpc/common/trpc_plugin.cc index dbbd226b..82d2e3b8 100644 --- a/trpc/common/trpc_plugin.cc +++ b/trpc/common/trpc_plugin.cc @@ -56,6 +56,7 @@ #include "trpc/transport/common/ssl_helper.h" #include "trpc/util/log/default/default_log.h" #include "trpc/util/net_util.h" +#include "trpc/naming/common/util/loadbalance/trpc_load_balance.h" namespace trpc { diff --git a/trpc/naming/common/util/hash/BUILD b/trpc/naming/common/util/hash/BUILD new file mode 100644 index 00000000..ce96b318 --- /dev/null +++ b/trpc/naming/common/util/hash/BUILD @@ -0,0 +1,49 @@ +# Description: trpc-cpp. + +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "hash_func", + srcs = ["hash_func.cc"], + hdrs = ["hash_func.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + "//trpc/naming/common/util/hash:city", + "//trpc/naming/common/util/hash:md5", + "//trpc/naming/common/util/hash:murmurhash3", + ], +) + +cc_library( + name = "murmurhash3", + srcs = ["murmurhash3.cc"], + hdrs = ["murmurhash3.h"], + visibility = [ + "//visibility:public", + ], + deps = [], +) + +cc_library( + name = "city", + srcs = ["city.cc"], + hdrs = ["city.h"], + visibility = [ + "//visibility:public", + ], + deps = [], +) + +cc_library( + name = "md5", + srcs = ["md5.cc"], + hdrs = ["md5.h"], + visibility = [ + "//visibility:public", + ], + deps = [], +) diff --git a/trpc/naming/common/util/hash/city.cc b/trpc/naming/common/util/hash/city.cc new file mode 100755 index 00000000..0ddf9b77 --- /dev/null +++ b/trpc/naming/common/util/hash/city.cc @@ -0,0 +1,452 @@ +// +// +// Tencent is pleased to support the open source community by making tRPC available. +// +// Copyright (C) 2024 THL A29 Limited, a Tencent company. +// All rights reserved. +// +// If you have downloaded a copy of the tRPC source code from Tencent, +// please note that tRPC source code is licensed under the Apache 2.0 License, +// A copy of the Apache 2.0 License is included in this file. +// +// tRPC is licensed under the Apache 2.0 License, and includes source codes from +// the following components: +// 1. smhasher +// +// Copyright (c) 2011 Google, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// +// CityHash, by Geoff Pike and Jyrki Alakuijala +// +// This file provides CityHash64() and related functions. +// +// It's probably possible to create even faster hash functions by +// writing a program that systematically explores some of the space of +// possible hash functions, by using SIMD instructions, or by +// compromising on hash quality. + +#include "trpc/naming/common/util/hash/city.h" + +#include // for memcpy and memset +#include + +using namespace std; + +static uint64 UNALIGNED_LOAD64(const char* p) { + uint64 result; + memcpy(&result, p, sizeof(result)); + return result; +} + +static uint32 UNALIGNED_LOAD32(const char* p) { + uint32 result; + memcpy(&result, p, sizeof(result)); + return result; +} + +#ifndef __BIG_ENDIAN__ + +#define uint32_in_expected_order(x) (x) +#define uint64_in_expected_order(x) (x) + +#else + +#ifdef _MSC_VER +#include +#define bswap_32(x) _byteswap_ulong(x) +#define bswap_64(x) _byteswap_uint64(x) + +#elif defined(__APPLE__) +// Mac OS X / Darwin features +#include +#define bswap_32(x) OSSwapInt32(x) +#define bswap_64(x) OSSwapInt64(x) + +#else +#include +#endif + +#define uint32_in_expected_order(x) (bswap_32(x)) +#define uint64_in_expected_order(x) (bswap_64(x)) + +#endif // __BIG_ENDIAN__ + +#if !defined(LIKELY) +#if defined(__GNUC__) || defined(__INTEL_COMPILER) +#define LIKELY(x) (__builtin_expect(!!(x), 1)) +#else +#define LIKELY(x) (x) +#endif +#endif + +static uint64 Fetch64(const char* p) { return uint64_in_expected_order(UNALIGNED_LOAD64(p)); } + +static uint32 Fetch32(const char* p) { return uint32_in_expected_order(UNALIGNED_LOAD32(p)); } + +// Some primes between 2^63 and 2^64 for various uses. +static const uint64 k0 = 0xc3a5c85c97cb3127ULL; +static const uint64 k1 = 0xb492b66fbe98f273ULL; +static const uint64 k2 = 0x9ae16a3b2f90404fULL; +static const uint64 k3 = 0xc949d7c7509e6557ULL; + +// Bitwise right rotate. Normally this will compile to a single +// instruction, especially if the shift is a manifest constant. +static uint64 Rotate(uint64 val, int shift) { + // Avoid shifting by 64: doing so yields an undefined result. + return shift == 0 ? val : ((val >> shift) | (val << (64 - shift))); +} + +// Equivalent to Rotate(), but requires the second arg to be non-zero. +// On x86-64, and probably others, it's possible for this to compile +// to a single instruction if both args are already in registers. +static uint64 RotateByAtLeast1(uint64 val, int shift) { return (val >> shift) | (val << (64 - shift)); } + +static uint64 ShiftMix(uint64 val) { return val ^ (val >> 47); } + +static uint64 HashLen16(uint64 u, uint64 v) { return Hash128to64(uint128(u, v)); } + +static uint64 HashLen0to16(const char* s, size_t len) { + if (len > 8) { + uint64 a = Fetch64(s); + uint64 b = Fetch64(s + len - 8); + return HashLen16(a, RotateByAtLeast1(b + len, len)) ^ b; + } + if (len >= 4) { + uint64 a = Fetch32(s); + return HashLen16(len + (a << 3), Fetch32(s + len - 4)); + } + if (len > 0) { + uint8 a = s[0]; + uint8 b = s[len >> 1]; + uint8 c = s[len - 1]; + uint32 y = static_cast(a) + (static_cast(b) << 8); + uint32 z = len + (static_cast(c) << 2); + return ShiftMix(y * k2 ^ z * k3) * k2; + } + return k2; +} + +// This probably works well for 16-byte strings as well, but it may be overkill +// in that case. +static uint64 HashLen17to32(const char* s, size_t len) { + uint64 a = Fetch64(s) * k1; + uint64 b = Fetch64(s + 8); + uint64 c = Fetch64(s + len - 8) * k2; + uint64 d = Fetch64(s + len - 16) * k0; + return HashLen16(Rotate(a - b, 43) + Rotate(c, 30) + d, a + Rotate(b ^ k3, 20) - c + len); +} + +// Return a 16-byte hash for 48 bytes. Quick and dirty. +// Callers do best to use "random-looking" values for a and b. +static pair WeakHashLen32WithSeeds(uint64 w, uint64 x, uint64 y, uint64 z, uint64 a, uint64 b) { + a += w; + b = Rotate(b + a + z, 21); + uint64 c = a; + a += x; + a += y; + b += Rotate(a, 44); + return make_pair(a + z, b + c); +} + +// Return a 16-byte hash for s[0] ... s[31], a, and b. Quick and dirty. +static pair WeakHashLen32WithSeeds(const char* s, uint64 a, uint64 b) { + return WeakHashLen32WithSeeds(Fetch64(s), Fetch64(s + 8), Fetch64(s + 16), Fetch64(s + 24), a, b); +} + +// Return an 8-byte hash for 33 to 64 bytes. +static uint64 HashLen33to64(const char* s, size_t len) { + uint64 z = Fetch64(s + 24); + uint64 a = Fetch64(s) + (len + Fetch64(s + len - 16)) * k0; + uint64 b = Rotate(a + z, 52); + uint64 c = Rotate(a, 37); + a += Fetch64(s + 8); + c += Rotate(a, 7); + a += Fetch64(s + 16); + uint64 vf = a + z; + uint64 vs = b + Rotate(a, 31) + c; + a = Fetch64(s + 16) + Fetch64(s + len - 32); + z = Fetch64(s + len - 8); + b = Rotate(a + z, 52); + c = Rotate(a, 37); + a += Fetch64(s + len - 24); + c += Rotate(a, 7); + a += Fetch64(s + len - 16); + uint64 wf = a + z; + uint64 ws = b + Rotate(a, 31) + c; + uint64 r = ShiftMix((vf + ws) * k2 + (wf + vs) * k0); + return ShiftMix(r * k0 + vs) * k2; +} + +uint64 CityHash64(const char* s, size_t len) { + if (len <= 32) { + if (len <= 16) { + return HashLen0to16(s, len); + } else { + return HashLen17to32(s, len); + } + } else if (len <= 64) { + return HashLen33to64(s, len); + } + + // For strings over 64 bytes we hash the end first, and then as we + // loop we keep 56 bytes of state: v, w, x, y, and z. + uint64 x = Fetch64(s + len - 40); + uint64 y = Fetch64(s + len - 16) + Fetch64(s + len - 56); + uint64 z = HashLen16(Fetch64(s + len - 48) + len, Fetch64(s + len - 24)); + pair v = WeakHashLen32WithSeeds(s + len - 64, len, z); + pair w = WeakHashLen32WithSeeds(s + len - 32, y + k1, x); + x = x * k1 + Fetch64(s); + + // Decrease len to the nearest multiple of 64, and operate on 64-byte chunks. + len = (len - 1) & ~static_cast(63); + do { + x = Rotate(x + y + v.first + Fetch64(s + 8), 37) * k1; + y = Rotate(y + v.second + Fetch64(s + 48), 42) * k1; + x ^= w.second; + y += v.first + Fetch64(s + 40); + z = Rotate(z + w.first, 33) * k1; + v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first); + w = WeakHashLen32WithSeeds(s + 32, z + w.second, y + Fetch64(s + 16)); + std::swap(z, x); + s += 64; + len -= 64; + } while (len != 0); + return HashLen16(HashLen16(v.first, w.first) + ShiftMix(y) * k1 + z, HashLen16(v.second, w.second) + x); +} + +uint64 CityHash64WithSeed(const char* s, size_t len, uint64 seed) { return CityHash64WithSeeds(s, len, k2, seed); } + +uint64 CityHash64WithSeeds(const char* s, size_t len, uint64 seed0, uint64 seed1) { + return HashLen16(CityHash64(s, len) - seed0, seed1); +} + +// A subroutine for CityHash128(). Returns a decent 128-bit hash for strings +// of any length representable in signed long. Based on City and Murmur. +static uint128 CityMurmur(const char* s, size_t len, uint128 seed) { + uint64 a = Uint128Low64(seed); + uint64 b = Uint128High64(seed); + uint64 c = 0; + uint64 d = 0; + signed long l = len - 16; + if (l <= 0) { // len <= 16 + a = ShiftMix(a * k1) * k1; + c = b * k1 + HashLen0to16(s, len); + d = ShiftMix(a + (len >= 8 ? Fetch64(s) : c)); + } else { // len > 16 + c = HashLen16(Fetch64(s + len - 8) + k1, a); + d = HashLen16(b + len, c + Fetch64(s + len - 16)); + a += d; + do { + a ^= ShiftMix(Fetch64(s) * k1) * k1; + a *= k1; + b ^= a; + c ^= ShiftMix(Fetch64(s + 8) * k1) * k1; + c *= k1; + d ^= c; + s += 16; + l -= 16; + } while (l > 0); + } + a = HashLen16(a, c); + b = HashLen16(d, b); + return uint128(a ^ b, HashLen16(b, a)); +} + +uint128 CityHash128WithSeed(const char* s, size_t len, uint128 seed) { + if (len < 128) { + return CityMurmur(s, len, seed); + } + + // We expect len >= 128 to be the common case. Keep 56 bytes of state: + // v, w, x, y, and z. + pair v, w; + uint64 x = Uint128Low64(seed); + uint64 y = Uint128High64(seed); + uint64 z = len * k1; + v.first = Rotate(y ^ k1, 49) * k1 + Fetch64(s); + v.second = Rotate(v.first, 42) * k1 + Fetch64(s + 8); + w.first = Rotate(y + z, 35) * k1 + x; + w.second = Rotate(x + Fetch64(s + 88), 53) * k1; + + // This is the same inner loop as CityHash64(), manually unrolled. + do { + x = Rotate(x + y + v.first + Fetch64(s + 8), 37) * k1; + y = Rotate(y + v.second + Fetch64(s + 48), 42) * k1; + x ^= w.second; + y += v.first + Fetch64(s + 40); + z = Rotate(z + w.first, 33) * k1; + v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first); + w = WeakHashLen32WithSeeds(s + 32, z + w.second, y + Fetch64(s + 16)); + std::swap(z, x); + s += 64; + x = Rotate(x + y + v.first + Fetch64(s + 8), 37) * k1; + y = Rotate(y + v.second + Fetch64(s + 48), 42) * k1; + x ^= w.second; + y += v.first + Fetch64(s + 40); + z = Rotate(z + w.first, 33) * k1; + v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first); + w = WeakHashLen32WithSeeds(s + 32, z + w.second, y + Fetch64(s + 16)); + std::swap(z, x); + s += 64; + len -= 128; + } while (LIKELY(len >= 128)); + x += Rotate(v.first + z, 49) * k0; + z += Rotate(w.first, 37) * k0; + // If 0 < len < 128, hash up to 4 chunks of 32 bytes each from the end of s. + for (size_t tail_done = 0; tail_done < len;) { + tail_done += 32; + y = Rotate(x + y, 42) * k0 + v.second; + w.first += Fetch64(s + len - tail_done + 16); + x = x * k0 + w.first; + z += w.second + Fetch64(s + len - tail_done); + w.second += v.first; + v = WeakHashLen32WithSeeds(s + len - tail_done, v.first + z, v.second); + } + // At this point our 56 bytes of state should contain more than + // enough information for a strong 128-bit hash. We use two + // different 56-byte-to-8-byte hashes to get a 16-byte final result. + x = HashLen16(x, v.first); + y = HashLen16(y + z, w.first); + return uint128(HashLen16(x + v.second, w.second) + y, HashLen16(x + w.second, y + v.second)); +} + +uint128 CityHash128(const char* s, size_t len) { + if (len >= 16) { + return CityHash128WithSeed(s + 16, len - 16, uint128(Fetch64(s) ^ k3, Fetch64(s + 8))); + } else if (len >= 8) { + return CityHash128WithSeed(NULL, 0, uint128(Fetch64(s) ^ (len * k0), Fetch64(s + len - 8) ^ k1)); + } else { + return CityHash128WithSeed(s, len, uint128(k0, k1)); + } +} + +#if defined(__SSE4_2__) && defined(__x86_64__) +#include + +// Requires len >= 240. +static void CityHashCrc256Long(const char* s, size_t len, uint32 seed, uint64* result) { + uint64 a = Fetch64(s + 56) + k0; + uint64 b = Fetch64(s + 96) + k0; + uint64 c = result[0] = HashLen16(b, len); + uint64 d = result[1] = Fetch64(s + 120) * k0 + len; + uint64 e = Fetch64(s + 184) + seed; + uint64 f = seed; + uint64 g = 0; + uint64 h = 0; + uint64 i = 0; + uint64 j = 0; + uint64 t = c + d; + + // 240 bytes of input per iter. + size_t iters = len / 240; + len -= iters * 240; + do { +#define CHUNK(multiplier, z) \ + { \ + uint64 old_a = a; \ + a = Rotate(b, 41 ^ z) * multiplier + Fetch64(s); \ + b = Rotate(c, 27 ^ z) * multiplier + Fetch64(s + 8); \ + c = Rotate(d, 41 ^ z) * multiplier + Fetch64(s + 16); \ + d = Rotate(e, 33 ^ z) * multiplier + Fetch64(s + 24); \ + e = Rotate(t, 25 ^ z) * multiplier + Fetch64(s + 32); \ + t = old_a; \ + } \ + f = _mm_crc32_u64(f, a); \ + g = _mm_crc32_u64(g, b); \ + h = _mm_crc32_u64(h, c); \ + i = _mm_crc32_u64(i, d); \ + j = _mm_crc32_u64(j, e); \ + s += 40 + + CHUNK(1, 1); + CHUNK(k0, 0); + CHUNK(1, 1); + CHUNK(k0, 0); + CHUNK(1, 1); + CHUNK(k0, 0); + } while (--iters > 0); + + while (len >= 40) { + CHUNK(k0, 0); + len -= 40; + } + if (len > 0) { + s = s + len - 40; + CHUNK(k0, 0); + } + j += i << 32; + a = HashLen16(a, j); + h += g << 32; + b += h; + c = HashLen16(c, f) + i; + d = HashLen16(d, e + result[0]); + j += e; + i += HashLen16(h, t); + e = HashLen16(a, d) + j; + f = HashLen16(b, c) + a; + g = HashLen16(j, i) + c; + result[0] = e + f + g + h; + a = ShiftMix((a + g) * k0) * k0 + b; + result[1] += a + result[0]; + a = ShiftMix(a * k0) * k0 + c; + result[2] = a + result[1]; + a = ShiftMix((a + e) * k0) * k0; + result[3] = a + result[2]; +} + +// Requires len < 240. +static void CityHashCrc256Short(const char* s, size_t len, uint64* result) { + char buf[240]; + memcpy(buf, s, len); + memset(buf + len, 0, 240 - len); + CityHashCrc256Long(buf, 240, ~static_cast(len), result); +} + +void CityHashCrc256(const char* s, size_t len, uint64* result) { + if (LIKELY(len >= 240)) { + CityHashCrc256Long(s, len, 0, result); + } else { + CityHashCrc256Short(s, len, result); + } +} + +uint128 CityHashCrc128WithSeed(const char* s, size_t len, uint128 seed) { + if (len <= 900) { + return CityHash128WithSeed(s, len, seed); + } else { + uint64 result[4]; + CityHashCrc256(s, len, result); + uint64 u = Uint128High64(seed) + result[0]; + uint64 v = Uint128Low64(seed) + result[1]; + return uint128(HashLen16(u, v + result[2]), HashLen16(Rotate(v, 32), u * k0 + result[3])); + } +} + +uint128 CityHashCrc128(const char* s, size_t len) { + if (len <= 900) { + return CityHash128(s, len); + } else { + uint64 result[4]; + CityHashCrc256(s, len, result); + return uint128(result[2], result[3]); + } +} + +#endif diff --git a/trpc/naming/common/util/hash/city.h b/trpc/naming/common/util/hash/city.h new file mode 100755 index 00000000..2ed1b29f --- /dev/null +++ b/trpc/naming/common/util/hash/city.h @@ -0,0 +1,125 @@ +// +// +// Tencent is pleased to support the open source community by making tRPC available. +// +// Copyright (C) 2024 THL A29 Limited, a Tencent company. +// All rights reserved. +// +// If you have downloaded a copy of the tRPC source code from Tencent, +// please note that tRPC source code is licensed under the Apache 2.0 License, +// A copy of the Apache 2.0 License is included in this file. +// +// tRPC is licensed under the Apache 2.0 License, and includes source codes from +// the following components: +// 1. smhasher +// +// Copyright (c) 2011 Google, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// +// CityHash, by Geoff Pike and Jyrki Alakuijala +// +// This file provides a few functions for hashing strings. On x86-64 +// hardware in 2011, CityHash64() is faster than other high-quality +// hash functions, such as Murmur. This is largely due to higher +// instruction-level parallelism. CityHash64() and CityHash128() also perform +// well on hash-quality tests. +// +// CityHash128() is optimized for relatively long strings and returns +// a 128-bit hash. For strings more than about 2000 bytes it can be +// faster than CityHash64(). +// +// Functions in the CityHash family are not suitable for cryptography. +// +// WARNING: This code has not been tested on big-endian platforms! +// It is known to work well on little-endian platforms that have a small penalty +// for unaligned reads, such as current Intel and AMD moderate-to-high-end CPUs. +// +// By the way, for some hash functions, given strings a and b, the hash +// of a+b is easily derived from the hashes of a and b. This property +// doesn't hold for any hash functions in this file. + +#pragma once + +#include // for size_t. +#include + +// Microsoft Visual Studio may not have stdint.h. +#if defined(_MSC_VER) && (_MSC_VER < 1600) +typedef unsigned char uint8_t; +typedef unsigned int uint32_t; +typedef unsigned __int64 uint64_t; +#else // defined(_MSC_VER) +#include +#endif // !defined(_MSC_VER) + +typedef uint8_t uint8; +typedef uint32_t uint32; +typedef uint64_t uint64; +typedef std::pair uint128; + +inline uint64 Uint128Low64(const uint128& x) { return x.first; } +inline uint64 Uint128High64(const uint128& x) { return x.second; } + +// Hash function for a byte array. +uint64 CityHash64(const char* buf, size_t len); + +// Hash function for a byte array. For convenience, a 64-bit seed is also +// hashed into the result. +uint64 CityHash64WithSeed(const char* buf, size_t len, uint64 seed); + +// Hash function for a byte array. For convenience, two seeds are also +// hashed into the result. +uint64 CityHash64WithSeeds(const char* buf, size_t len, uint64 seed0, uint64 seed1); + +// Hash function for a byte array. +uint128 CityHash128(const char* s, size_t len); + +// Hash function for a byte array. For convenience, a 128-bit seed is also +// hashed into the result. +uint128 CityHash128WithSeed(const char* s, size_t len, uint128 seed); + +// Hash 128 input bits down to 64 bits of output. +// This is intended to be a reasonably good hash function. +inline uint64 Hash128to64(const uint128& x) { + // Murmur-inspired hashing. + const uint64 kMul = 0x9ddfea08eb382d69ULL; + uint64 a = (Uint128Low64(x) ^ Uint128High64(x)) * kMul; + a ^= (a >> 47); + uint64 b = (Uint128High64(x) ^ a) * kMul; + b ^= (b >> 47); + b *= kMul; + return b; +} + +// Conditionally include declarations for versions of City that require SSE4.2 +// instructions to be available. +#if defined(__SSE4_2__) && defined(__x86_64__) + +// Hash function for a byte array. +uint128 CityHashCrc128(const char* s, size_t len); + +// Hash function for a byte array. For convenience, a 128-bit seed is also +// hashed into the result. +uint128 CityHashCrc128WithSeed(const char* s, size_t len, uint128 seed); + +// Hash function for a byte array. Sets result[0] ... result[3]. +void CityHashCrc256(const char* s, size_t len, uint64* result); + +#endif // __SSE4_2__ diff --git a/trpc/naming/common/util/hash/hash_func.cc b/trpc/naming/common/util/hash/hash_func.cc new file mode 100644 index 00000000..199e3e4f --- /dev/null +++ b/trpc/naming/common/util/hash/hash_func.cc @@ -0,0 +1,103 @@ +// +// +// Tencent is pleased to support the open source community by making tRPC available. +// +// Copyright (C) 2024 THL A29 Limited, a Tencent company. +// All rights reserved. +// +// If you have downloaded a copy of the tRPC source code from Tencent, +// please note that tRPC source code is licensed under the Apache 2.0 License, +// A copy of the Apache 2.0 License is included in this file. +// +// + +#include "trpc/naming/common/util/hash/hash_func.h" + +#include +#include + +#include "trpc/naming/common/util/hash/city.h" +#include "trpc/naming/common/util/hash/md5.h" +#include "trpc/naming/common/util/hash/murmurhash3.h" + +namespace trpc { + +std::uint64_t Md5Hash(const std::string& input) { + const uint32_t seed = 131; + uint64_t hash = Md5Hash_32(input.c_str(), input.size(), seed); + return hash; +} +std::uint64_t BkdrHash(const std::string& input) { + const uint64_t seed = 131; + uint64_t hash = 0; + for (char ch : input) { + hash = hash * seed + static_cast(ch); + } + return hash; +} +std::uint64_t Fnv1aHash(const std::string& input) { + const uint64_t fnv_prime = 0x811C9DC5; + size_t hash = 0; + for (char ch : input) { + hash ^= static_cast(ch); + hash *= fnv_prime; + } + return hash; +} + +std::uint64_t MurmurHash3(const std::string& input) { + uint32_t seed = 42; + uint64_t hash[2]; + MurmurHash3_x64_128(input.c_str(), input.size(), seed, hash); + return hash[0]; +} + +std::uint64_t CityHash(const std::string& input) { + uint64_t seed = 42; + return CityHash64WithSeed(input.c_str(), input.size(), seed); +} + +std::uint64_t GetHash(const std::string& input, const HashFuncName& hash_func) { + uint64_t hash = 0; + switch (hash_func) { + case HashFuncName::kMd5: + hash = Md5Hash(input); + break; + case HashFuncName::kBkdr: + hash = BkdrHash(input); + break; + case HashFuncName::kFnv1a: + hash = Fnv1aHash(input); + break; + case HashFuncName::kMurmur3: + hash = MurmurHash3(input); + break; + case HashFuncName::kCity: + hash = CityHash(input); + break; + default: + hash = MurmurHash3(input); + break; + } + return hash; +} + +std::uint64_t Hash(const std::string& input, const std::string& hash_func) { + HashFuncName func = + kHashFuncTable.find(hash_func) == kHashFuncTable.end() ? HashFuncName::kDefault : kHashFuncTable.at(hash_func); + return GetHash(input, func); +} + +std::uint64_t Hash(const std::string& input, const HashFuncName& hash_func) { return GetHash(input, hash_func); } + +std::uint64_t Hash(const std::string& input, const std::string& hash_func, uint64_t num) { + HashFuncName func = + kHashFuncTable.find(hash_func) == kHashFuncTable.end() ? HashFuncName::kDefault : kHashFuncTable.at(hash_func); + return GetHash(input, func) % num; +} + +std::uint64_t Hash(const std::string& input, const HashFuncName& hash_func, uint64_t num) { + return GetHash(input, hash_func) % num; +} + +} // namespace trpc \ No newline at end of file diff --git a/trpc/naming/common/util/hash/hash_func.h b/trpc/naming/common/util/hash/hash_func.h new file mode 100644 index 00000000..dc0c2c00 --- /dev/null +++ b/trpc/naming/common/util/hash/hash_func.h @@ -0,0 +1,58 @@ +// +// +// Tencent is pleased to support the open source community by making tRPC available. +// +// Copyright (C) 2024 THL A29 Limited, a Tencent company. +// All rights reserved. +// +// If you have downloaded a copy of the tRPC source code from Tencent, +// please note that tRPC source code is licensed under the Apache 2.0 License, +// A copy of the Apache 2.0 License is included in this file. +// +// + +#pragma once + +#include +#include +#include + +namespace trpc { + +static constexpr int kHashNodesMaxIndex = 5; + +enum HashFuncName { + kDefault, + kMd5, + kBkdr, + kFnv1a, + kMurmur3, + kCity, +}; + +static const std::unordered_map kHashFuncTable = { + {"md5", HashFuncName::kMd5}, {"bkdr", HashFuncName::kBkdr}, {"fnv1a", HashFuncName::kFnv1a}, + {"murmur3", HashFuncName::kMurmur3}, {"city", HashFuncName::kCity}, +}; + +std::uint64_t Md5Hash(const std::string& input); + +std::uint64_t BkdrHash(const std::string& input); + +std::uint64_t Fnv1aHash(const std::string& input); + +std::uint64_t MurmurHash3(const std::string& input); + +std::uint64_t CityHash(const std::string& input); + +std::uint64_t GetHash(const std::string& input, const std::string& hash_func); + +std::uint64_t Hash(const std::string& input, const std::string& hash_func); + +std::uint64_t Hash(const std::string& input, const HashFuncName& hash_func); + +std::uint64_t Hash(const std::string& input, const std::string& hash_func, uint64_t num); + +std::uint64_t Hash(const std::string& input, const HashFuncName& hash_func, uint64_t num); + +} // namespace trpc \ No newline at end of file diff --git a/trpc/naming/common/util/hash/md5.cc b/trpc/naming/common/util/hash/md5.cc new file mode 100644 index 00000000..ed1b73e0 --- /dev/null +++ b/trpc/naming/common/util/hash/md5.cc @@ -0,0 +1,272 @@ +// +// +// Tencent is pleased to support the open source community by making tRPC available. +// +// Copyright (C) 2024 THL A29 Limited, a Tencent company. +// All rights reserved. +// +// If you have downloaded a copy of the tRPC source code from Tencent, +// please note that tRPC source code is licensed under the Apache 2.0 License, +// A copy of the Apache 2.0 License is included in this file. +// +// tRPC is licensed under the Apache 2.0 License, and includes source codes from +// the following components: +// 1. smhasher +// +// "Derived from the RSA Data Security, Inc. MD5 Message Digest Algorithm" +// Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All +// rights reserved. + +// License to copy and use this software is granted provided that it +// is identified as the "RSA Data Security, Inc. MD5 Message-Digest +// Algorithm" in all material mentioning or referencing this software +// or this function. + +// License is also granted to make and use derivative works provided +// that such works are identified as "derived from the RSA Data +// Security, Inc. MD5 Message-Digest Algorithm" in all material +// mentioning or referencing the derived work. + +// RSA Data Security, Inc. makes no representations concerning either +// the merchantability of this software or the suitability of this +// software for any particular purpose. It is provided "as is" +// without express or implied warranty of any kind. + +// These notices must be retained in any copies of any part of this +// documentation and/or software. +#include "trpc/naming/common/util/hash/md5.h" + +#include +#include +/* + * MD5 context setup + */ +void Md5Starts(Md5Context* ctx) { + ctx->total[0] = 0; + ctx->total[1] = 0; + + ctx->state[0] = 0x67452301; + ctx->state[1] = 0xEFCDAB89; + ctx->state[2] = 0x98BADCFE; + ctx->state[3] = 0x10325476; +} + +static void Md5Process(Md5Context* ctx, unsigned char data[64]) { + unsigned long X[16], A, B, C, D; + + GET_ULONG_LE(X[0], data, 0); + GET_ULONG_LE(X[1], data, 4); + GET_ULONG_LE(X[2], data, 8); + GET_ULONG_LE(X[3], data, 12); + GET_ULONG_LE(X[4], data, 16); + GET_ULONG_LE(X[5], data, 20); + GET_ULONG_LE(X[6], data, 24); + GET_ULONG_LE(X[7], data, 28); + GET_ULONG_LE(X[8], data, 32); + GET_ULONG_LE(X[9], data, 36); + GET_ULONG_LE(X[10], data, 40); + GET_ULONG_LE(X[11], data, 44); + GET_ULONG_LE(X[12], data, 48); + GET_ULONG_LE(X[13], data, 52); + GET_ULONG_LE(X[14], data, 56); + GET_ULONG_LE(X[15], data, 60); + +#define S(x, n) ((x << n) | ((x & 0xFFFFFFFF) >> (32 - n))) + +#define P(a, b, c, d, k, s, t) \ + { \ + a += F(b, c, d) + X[k] + t; \ + a = S(a, s) + b; \ + } + + A = ctx->state[0]; + B = ctx->state[1]; + C = ctx->state[2]; + D = ctx->state[3]; + +#define F(x, y, z) (z ^ (x & (y ^ z))) + + P(A, B, C, D, 0, 7, 0xD76AA478); + P(D, A, B, C, 1, 12, 0xE8C7B756); + P(C, D, A, B, 2, 17, 0x242070DB); + P(B, C, D, A, 3, 22, 0xC1BDCEEE); + P(A, B, C, D, 4, 7, 0xF57C0FAF); + P(D, A, B, C, 5, 12, 0x4787C62A); + P(C, D, A, B, 6, 17, 0xA8304613); + P(B, C, D, A, 7, 22, 0xFD469501); + P(A, B, C, D, 8, 7, 0x698098D8); + P(D, A, B, C, 9, 12, 0x8B44F7AF); + P(C, D, A, B, 10, 17, 0xFFFF5BB1); + P(B, C, D, A, 11, 22, 0x895CD7BE); + P(A, B, C, D, 12, 7, 0x6B901122); + P(D, A, B, C, 13, 12, 0xFD987193); + P(C, D, A, B, 14, 17, 0xA679438E); + P(B, C, D, A, 15, 22, 0x49B40821); + +#undef F + +#define F(x, y, z) (y ^ (z & (x ^ y))) + + P(A, B, C, D, 1, 5, 0xF61E2562); + P(D, A, B, C, 6, 9, 0xC040B340); + P(C, D, A, B, 11, 14, 0x265E5A51); + P(B, C, D, A, 0, 20, 0xE9B6C7AA); + P(A, B, C, D, 5, 5, 0xD62F105D); + P(D, A, B, C, 10, 9, 0x02441453); + P(C, D, A, B, 15, 14, 0xD8A1E681); + P(B, C, D, A, 4, 20, 0xE7D3FBC8); + P(A, B, C, D, 9, 5, 0x21E1CDE6); + P(D, A, B, C, 14, 9, 0xC33707D6); + P(C, D, A, B, 3, 14, 0xF4D50D87); + P(B, C, D, A, 8, 20, 0x455A14ED); + P(A, B, C, D, 13, 5, 0xA9E3E905); + P(D, A, B, C, 2, 9, 0xFCEFA3F8); + P(C, D, A, B, 7, 14, 0x676F02D9); + P(B, C, D, A, 12, 20, 0x8D2A4C8A); + +#undef F + +#define F(x, y, z) (x ^ y ^ z) + + P(A, B, C, D, 5, 4, 0xFFFA3942); + P(D, A, B, C, 8, 11, 0x8771F681); + P(C, D, A, B, 11, 16, 0x6D9D6122); + P(B, C, D, A, 14, 23, 0xFDE5380C); + P(A, B, C, D, 1, 4, 0xA4BEEA44); + P(D, A, B, C, 4, 11, 0x4BDECFA9); + P(C, D, A, B, 7, 16, 0xF6BB4B60); + P(B, C, D, A, 10, 23, 0xBEBFBC70); + P(A, B, C, D, 13, 4, 0x289B7EC6); + P(D, A, B, C, 0, 11, 0xEAA127FA); + P(C, D, A, B, 3, 16, 0xD4EF3085); + P(B, C, D, A, 6, 23, 0x04881D05); + P(A, B, C, D, 9, 4, 0xD9D4D039); + P(D, A, B, C, 12, 11, 0xE6DB99E5); + P(C, D, A, B, 15, 16, 0x1FA27CF8); + P(B, C, D, A, 2, 23, 0xC4AC5665); + +#undef F + +#define F(x, y, z) (y ^ (x | ~z)) + + P(A, B, C, D, 0, 6, 0xF4292244); + P(D, A, B, C, 7, 10, 0x432AFF97); + P(C, D, A, B, 14, 15, 0xAB9423A7); + P(B, C, D, A, 5, 21, 0xFC93A039); + P(A, B, C, D, 12, 6, 0x655B59C3); + P(D, A, B, C, 3, 10, 0x8F0CCC92); + P(C, D, A, B, 10, 15, 0xFFEFF47D); + P(B, C, D, A, 1, 21, 0x85845DD1); + P(A, B, C, D, 8, 6, 0x6FA87E4F); + P(D, A, B, C, 15, 10, 0xFE2CE6E0); + P(C, D, A, B, 6, 15, 0xA3014314); + P(B, C, D, A, 13, 21, 0x4E0811A1); + P(A, B, C, D, 4, 6, 0xF7537E82); + P(D, A, B, C, 11, 10, 0xBD3AF235); + P(C, D, A, B, 2, 15, 0x2AD7D2BB); + P(B, C, D, A, 9, 21, 0xEB86D391); + +#undef F + + ctx->state[0] += A; + ctx->state[1] += B; + ctx->state[2] += C; + ctx->state[3] += D; +} + +/* + * MD5 process buffer + */ +void Md5Update(Md5Context* ctx, unsigned char* input, int ilen) { + int fill; + unsigned long left; + + if (ilen <= 0) return; + + left = ctx->total[0] & 0x3F; + fill = 64 - left; + + ctx->total[0] += ilen; + ctx->total[0] &= 0xFFFFFFFF; + + if (ctx->total[0] < (unsigned long)ilen) ctx->total[1]++; + + if (left && ilen >= fill) { + memcpy((void*)(ctx->buffer + left), (void*)input, fill); + Md5Process(ctx, ctx->buffer); + input += fill; + ilen -= fill; + left = 0; + } + + while (ilen >= 64) { + Md5Process(ctx, input); + input += 64; + ilen -= 64; + } + + if (ilen > 0) { + memcpy((void*)(ctx->buffer + left), (void*)input, ilen); + } +} + +static const unsigned char md5_padding[64] = {0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + +/* + * MD5 final digest + */ +void Md5Finish(Md5Context* ctx, unsigned char output[16]) { + unsigned long last, padn; + unsigned long high, low; + unsigned char msglen[8]; + + high = (ctx->total[0] >> 29) | (ctx->total[1] << 3); + low = (ctx->total[0] << 3); + + PUT_ULONG_LE(low, msglen, 0); + PUT_ULONG_LE(high, msglen, 4); + + last = ctx->total[0] & 0x3F; + padn = (last < 56) ? (56 - last) : (120 - last); + + Md5Update(ctx, (unsigned char*)md5_padding, padn); + Md5Update(ctx, msglen, 8); + + PUT_ULONG_LE(ctx->state[0], output, 0); + PUT_ULONG_LE(ctx->state[1], output, 4); + PUT_ULONG_LE(ctx->state[2], output, 8); + PUT_ULONG_LE(ctx->state[3], output, 12); +} + +/* + * output = MD5( input buffer ) + */ +void Md5(unsigned char* input, int ilen, unsigned char output[16]) { + Md5Context ctx; + + Md5Starts(&ctx); + Md5Update(&ctx, input, ilen); + Md5Finish(&ctx, output); + + memset(&ctx, 0, sizeof(Md5Context)); +} + +unsigned int Md5Hash_32(const void* input, int len, unsigned int /*seed*/) { + unsigned int hash[4]; + + Md5((unsigned char*)input, len, (unsigned char*)hash); + + // return hash[0] ^ hash[1] ^ hash[2] ^ hash[3]; + + return hash[0]; +} + +void Md5_32(const void* key, int len, uint32_t /*seed*/, void* out) { + unsigned int hash[4]; + + Md5((unsigned char*)key, len, (unsigned char*)hash); + + *(uint32_t*)out = hash[0]; +} \ No newline at end of file diff --git a/trpc/naming/common/util/hash/md5.h b/trpc/naming/common/util/hash/md5.h new file mode 100644 index 00000000..47e0dfbc --- /dev/null +++ b/trpc/naming/common/util/hash/md5.h @@ -0,0 +1,109 @@ +// +// +// Tencent is pleased to support the open source community by making tRPC available. +// +// Copyright (C) 2024 THL A29 Limited, a Tencent company. +// All rights reserved. +// +// If you have downloaded a copy of the tRPC source code from Tencent, +// please note that tRPC source code is licensed under the Apache 2.0 License, +// A copy of the Apache 2.0 License is included in this file. +// +// tRPC is licensed under the Apache 2.0 License, and includes source codes from +// the following components: +// 1. smhasher +// +// "Derived from the RSA Data Security, Inc. MD5 Message Digest Algorithm" +// Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All +// rights reserved. + +// License to copy and use this software is granted provided that it +// is identified as the "RSA Data Security, Inc. MD5 Message-Digest +// Algorithm" in all material mentioning or referencing this software +// or this function. + +// License is also granted to make and use derivative works provided +// that such works are identified as "derived from the RSA Data +// Security, Inc. MD5 Message-Digest Algorithm" in all material +// mentioning or referencing the derived work. + +// RSA Data Security, Inc. makes no representations concerning either +// the merchantability of this software or the suitability of this +// software for any particular purpose. It is provided "as is" +// without express or implied warranty of any kind. + +// These notices must be retained in any copies of any part of this +// documentation and/or software. +/* + * 32-bit integer manipulation macros (little endian) + */ + +#pragma once + +#include + +#ifndef GET_ULONG_LE +#define GET_ULONG_LE(n, b, i) \ + { \ + (n) = ((unsigned long)(b)[(i)]) | ((unsigned long)(b)[(i) + 1] << 8) | ((unsigned long)(b)[(i) + 2] << 16) | \ + ((unsigned long)(b)[(i) + 3] << 24); \ + } +#endif + +#ifndef PUT_ULONG_LE +#define PUT_ULONG_LE(n, b, i) \ + { \ + (b)[(i)] = (unsigned char)((n)); \ + (b)[(i) + 1] = (unsigned char)((n) >> 8); \ + (b)[(i) + 2] = (unsigned char)((n) >> 16); \ + (b)[(i) + 3] = (unsigned char)((n) >> 24); \ + } +#endif +/** + * \brief MD5 context structure + */ +typedef struct { + unsigned long total[2]; /*!< number of bytes processed */ + unsigned long state[4]; /*!< intermediate digest state */ + unsigned char buffer[64]; /*!< data block being processed */ + + unsigned char ipad[64]; /*!< HMAC: inner padding */ + unsigned char opad[64]; /*!< HMAC: outer padding */ +} Md5Context; + +/** + * \brief MD5 context setup + * + * \param ctx context to be initialized + */ +void Md5Starts(Md5Context* ctx); + +/** + * \brief MD5 process buffer + * + * \param ctx MD5 context + * \param input buffer holding the data + * \param ilen length of the input data + */ +void Md5Update(Md5Context* ctx, unsigned char* input, int ilen); + +/** + * \brief MD5 final digest + * + * \param ctx MD5 context + * \param output MD5 checksum result + */ +void Md5Finish(Md5Context* ctx, unsigned char output[16]); + +/** + * \brief Output = MD5( input buffer ) + * + * \param input buffer holding the data + * \param ilen length of the input data + * \param output MD5 checksum result + */ +void Md5(unsigned char* input, int ilen, unsigned char output[16]); + +unsigned int Md5Hash_32(const void* input, int len, unsigned int /*seed*/); + +void Md5_32(const void* key, int len, uint32_t /*seed*/, void* out); diff --git a/trpc/naming/common/util/hash/murmurhash3.cc b/trpc/naming/common/util/hash/murmurhash3.cc new file mode 100644 index 00000000..819000f3 --- /dev/null +++ b/trpc/naming/common/util/hash/murmurhash3.cc @@ -0,0 +1,419 @@ +// +// +// Tencent is pleased to support the open source community by making tRPC available. +// +// Copyright (C) 2024 THL A29 Limited, a Tencent company. +// All rights reserved. +// +// If you have downloaded a copy of the tRPC source code from Tencent, +// please note that tRPC source code is licensed under the Apache 2.0 License, +// A copy of the Apache 2.0 License is included in this file. +// +// tRPC is licensed under the Apache 2.0 License, and includes source codes from +// the following components: +// 1. smhasher +// +//----------------------------------------------------------------------------- +// MurmurHash3 was written by Austin Appleby, and is placed in the public +// domain. The author hereby disclaims copyright to this source code. + +// Note - The x86 and x64 versions do _not_ produce the same results, as the +// algorithms are optimized for their respective platforms. You can still +// compile and run any of them on any platform, but your performance with the +// non-native version will be less than optimal. + +#include "trpc/naming/common/util/hash/murmurhash3.h" + +//----------------------------------------------------------------------------- +// Platform-specific functions and macros + +// Microsoft Visual Studio + +#if defined(_MSC_VER) + +#define FORCE_INLINE __forceinline + +#include + +#define ROTL32(x, y) _rotl(x, y) +#define ROTL64(x, y) _rotl64(x, y) + +#define BIG_CONSTANT(x) (x) + +// Other compilers + +#else // defined(_MSC_VER) + +#define FORCE_INLINE inline __attribute__((always_inline)) + +inline uint32_t rotl32(uint32_t x, int8_t r) { return (x << r) | (x >> (32 - r)); } + +inline uint64_t rotl64(uint64_t x, int8_t r) { return (x << r) | (x >> (64 - r)); } + +#define ROTL32(x, y) rotl32(x, y) +#define ROTL64(x, y) rotl64(x, y) + +#define BIG_CONSTANT(x) (x##LLU) + +#endif // !defined(_MSC_VER) + +//----------------------------------------------------------------------------- +// Block read - if your platform needs to do endian-swapping or can only +// handle aligned reads, do the conversion here + +FORCE_INLINE uint32_t getblock32(const uint32_t* p, int i) { return p[i]; } + +FORCE_INLINE uint64_t getblock64(const uint64_t* p, int i) { return p[i]; } + +//----------------------------------------------------------------------------- +// Finalization mix - force all bits of a hash block to avalanche + +FORCE_INLINE uint32_t fmix32(uint32_t h) { + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + + return h; +} + +//---------- + +FORCE_INLINE uint64_t fmix64(uint64_t k) { + k ^= k >> 33; + k *= BIG_CONSTANT(0xff51afd7ed558ccd); + k ^= k >> 33; + k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53); + k ^= k >> 33; + + return k; +} + +//----------------------------------------------------------------------------- + +void MurmurHash3_x86_32(const void* key, int len, uint32_t seed, void* out) { + const uint8_t* data = (const uint8_t*)key; + const int nblocks = len / 4; + + uint32_t h1 = seed; + + const uint32_t c1 = 0xcc9e2d51; + const uint32_t c2 = 0x1b873593; + + //---------- + // body + + const uint32_t* blocks = (const uint32_t*)(data + nblocks * 4); + + for (int i = -nblocks; i; i++) { + uint32_t k1 = getblock32(blocks, i); + + k1 *= c1; + k1 = ROTL32(k1, 15); + k1 *= c2; + + h1 ^= k1; + h1 = ROTL32(h1, 13); + h1 = h1 * 5 + 0xe6546b64; + } + + //---------- + // tail + + const uint8_t* tail = (const uint8_t*)(data + nblocks * 4); + + uint32_t k1 = 0; + + switch (len & 3) { + case 3: + k1 ^= tail[2] << 16; + case 2: + k1 ^= tail[1] << 8; + case 1: + k1 ^= tail[0]; + k1 *= c1; + k1 = ROTL32(k1, 15); + k1 *= c2; + h1 ^= k1; + }; + + //---------- + // finalization + + h1 ^= len; + + h1 = fmix32(h1); + + *(uint32_t*)out = h1; +} + +//----------------------------------------------------------------------------- + +void MurmurHash3_x86_128(const void* key, const int len, uint32_t seed, void* out) { + const uint8_t* data = (const uint8_t*)key; + const int nblocks = len / 16; + + uint32_t h1 = seed; + uint32_t h2 = seed; + uint32_t h3 = seed; + uint32_t h4 = seed; + + const uint32_t c1 = 0x239b961b; + const uint32_t c2 = 0xab0e9789; + const uint32_t c3 = 0x38b34ae5; + const uint32_t c4 = 0xa1e38b93; + + //---------- + // body + + const uint32_t* blocks = (const uint32_t*)(data + nblocks * 16); + + for (int i = -nblocks; i; i++) { + uint32_t k1 = getblock32(blocks, i * 4 + 0); + uint32_t k2 = getblock32(blocks, i * 4 + 1); + uint32_t k3 = getblock32(blocks, i * 4 + 2); + uint32_t k4 = getblock32(blocks, i * 4 + 3); + + k1 *= c1; + k1 = ROTL32(k1, 15); + k1 *= c2; + h1 ^= k1; + + h1 = ROTL32(h1, 19); + h1 += h2; + h1 = h1 * 5 + 0x561ccd1b; + + k2 *= c2; + k2 = ROTL32(k2, 16); + k2 *= c3; + h2 ^= k2; + + h2 = ROTL32(h2, 17); + h2 += h3; + h2 = h2 * 5 + 0x0bcaa747; + + k3 *= c3; + k3 = ROTL32(k3, 17); + k3 *= c4; + h3 ^= k3; + + h3 = ROTL32(h3, 15); + h3 += h4; + h3 = h3 * 5 + 0x96cd1c35; + + k4 *= c4; + k4 = ROTL32(k4, 18); + k4 *= c1; + h4 ^= k4; + + h4 = ROTL32(h4, 13); + h4 += h1; + h4 = h4 * 5 + 0x32ac3b17; + } + + //---------- + // tail + + const uint8_t* tail = (const uint8_t*)(data + nblocks * 16); + + uint32_t k1 = 0; + uint32_t k2 = 0; + uint32_t k3 = 0; + uint32_t k4 = 0; + + switch (len & 15) { + case 15: + k4 ^= tail[14] << 16; + case 14: + k4 ^= tail[13] << 8; + case 13: + k4 ^= tail[12] << 0; + k4 *= c4; + k4 = ROTL32(k4, 18); + k4 *= c1; + h4 ^= k4; + + case 12: + k3 ^= tail[11] << 24; + case 11: + k3 ^= tail[10] << 16; + case 10: + k3 ^= tail[9] << 8; + case 9: + k3 ^= tail[8] << 0; + k3 *= c3; + k3 = ROTL32(k3, 17); + k3 *= c4; + h3 ^= k3; + + case 8: + k2 ^= tail[7] << 24; + case 7: + k2 ^= tail[6] << 16; + case 6: + k2 ^= tail[5] << 8; + case 5: + k2 ^= tail[4] << 0; + k2 *= c2; + k2 = ROTL32(k2, 16); + k2 *= c3; + h2 ^= k2; + + case 4: + k1 ^= tail[3] << 24; + case 3: + k1 ^= tail[2] << 16; + case 2: + k1 ^= tail[1] << 8; + case 1: + k1 ^= tail[0] << 0; + k1 *= c1; + k1 = ROTL32(k1, 15); + k1 *= c2; + h1 ^= k1; + }; + + //---------- + // finalization + + h1 ^= len; + h2 ^= len; + h3 ^= len; + h4 ^= len; + + h1 += h2; + h1 += h3; + h1 += h4; + h2 += h1; + h3 += h1; + h4 += h1; + + h1 = fmix32(h1); + h2 = fmix32(h2); + h3 = fmix32(h3); + h4 = fmix32(h4); + + h1 += h2; + h1 += h3; + h1 += h4; + h2 += h1; + h3 += h1; + h4 += h1; + + ((uint32_t*)out)[0] = h1; + ((uint32_t*)out)[1] = h2; + ((uint32_t*)out)[2] = h3; + ((uint32_t*)out)[3] = h4; +} + +//----------------------------------------------------------------------------- + +void MurmurHash3_x64_128(const void* key, const int len, const uint32_t seed, void* out) { + const uint8_t* data = (const uint8_t*)key; + const int nblocks = len / 16; + + uint64_t h1 = seed; + uint64_t h2 = seed; + + const uint64_t c1 = BIG_CONSTANT(0x87c37b91114253d5); + const uint64_t c2 = BIG_CONSTANT(0x4cf5ad432745937f); + + //---------- + // body + + const uint64_t* blocks = (const uint64_t*)(data); + + for (int i = 0; i < nblocks; i++) { + uint64_t k1 = getblock64(blocks, i * 2 + 0); + uint64_t k2 = getblock64(blocks, i * 2 + 1); + + k1 *= c1; + k1 = ROTL64(k1, 31); + k1 *= c2; + h1 ^= k1; + + h1 = ROTL64(h1, 27); + h1 += h2; + h1 = h1 * 5 + 0x52dce729; + + k2 *= c2; + k2 = ROTL64(k2, 33); + k2 *= c1; + h2 ^= k2; + + h2 = ROTL64(h2, 31); + h2 += h1; + h2 = h2 * 5 + 0x38495ab5; + } + + //---------- + // tail + + const uint8_t* tail = (const uint8_t*)(data + nblocks * 16); + + uint64_t k1 = 0; + uint64_t k2 = 0; + + switch (len & 15) { + case 15: + k2 ^= ((uint64_t)tail[14]) << 48; + case 14: + k2 ^= ((uint64_t)tail[13]) << 40; + case 13: + k2 ^= ((uint64_t)tail[12]) << 32; + case 12: + k2 ^= ((uint64_t)tail[11]) << 24; + case 11: + k2 ^= ((uint64_t)tail[10]) << 16; + case 10: + k2 ^= ((uint64_t)tail[9]) << 8; + case 9: + k2 ^= ((uint64_t)tail[8]) << 0; + k2 *= c2; + k2 = ROTL64(k2, 33); + k2 *= c1; + h2 ^= k2; + + case 8: + k1 ^= ((uint64_t)tail[7]) << 56; + case 7: + k1 ^= ((uint64_t)tail[6]) << 48; + case 6: + k1 ^= ((uint64_t)tail[5]) << 40; + case 5: + k1 ^= ((uint64_t)tail[4]) << 32; + case 4: + k1 ^= ((uint64_t)tail[3]) << 24; + case 3: + k1 ^= ((uint64_t)tail[2]) << 16; + case 2: + k1 ^= ((uint64_t)tail[1]) << 8; + case 1: + k1 ^= ((uint64_t)tail[0]) << 0; + k1 *= c1; + k1 = ROTL64(k1, 31); + k1 *= c2; + h1 ^= k1; + }; + + //---------- + // finalization + + h1 ^= len; + h2 ^= len; + + h1 += h2; + h2 += h1; + + h1 = fmix64(h1); + h2 = fmix64(h2); + + h1 += h2; + h2 += h1; + + ((uint64_t*)out)[0] = h1; + ((uint64_t*)out)[1] = h2; +} + +//----------------------------------------------------------------------------- diff --git a/trpc/naming/common/util/hash/murmurhash3.h b/trpc/naming/common/util/hash/murmurhash3.h new file mode 100644 index 00000000..9b8e4b53 --- /dev/null +++ b/trpc/naming/common/util/hash/murmurhash3.h @@ -0,0 +1,49 @@ +// +// +// Tencent is pleased to support the open source community by making tRPC available. +// +// Copyright (C) 2024 THL A29 Limited, a Tencent company. +// All rights reserved. +// +// If you have downloaded a copy of the tRPC source code from Tencent, +// please note that tRPC source code is licensed under the Apache 2.0 License, +// A copy of the Apache 2.0 License is included in this file. +// +// tRPC is licensed under the Apache 2.0 License, and includes source codes from +// the following components: +// 1. smhasher +// +//----------------------------------------------------------------------------- +// MurmurHash3 was written by Austin Appleby, and is placed in the public +// domain. The author hereby disclaims copyright to this source code. + +#pragma once + +//----------------------------------------------------------------------------- +// Platform-specific functions and macros + +// Microsoft Visual Studio + +#if defined(_MSC_VER) && (_MSC_VER < 1600) + +typedef unsigned char uint8_t; +typedef unsigned int uint32_t; +typedef unsigned __int64 uint64_t; + +// Other compilers + +#else // defined(_MSC_VER) + +#include + +#endif // !defined(_MSC_VER) + +//----------------------------------------------------------------------------- + +void MurmurHash3_x86_32(const void* key, int len, uint32_t seed, void* out); + +void MurmurHash3_x86_128(const void* key, int len, uint32_t seed, void* out); + +void MurmurHash3_x64_128(const void* key, int len, uint32_t seed, void* out); + +//----------------------------------------------------------------------------- diff --git a/trpc/naming/common/util/loadbalance/BUILD b/trpc/naming/common/util/loadbalance/BUILD index 5095b903..532c07a7 100644 --- a/trpc/naming/common/util/loadbalance/BUILD +++ b/trpc/naming/common/util/loadbalance/BUILD @@ -10,6 +10,9 @@ cc_library( hdrs = ["trpc_load_balance.h"], deps = [ "//trpc/naming:load_balance_factory", + "//trpc/naming/common/util/loadbalance/hash:consistenthash_load_balance", + "//trpc/naming/common/util/loadbalance/hash:modulohash_load_balance", + "//trpc/naming/common/util/loadbalance/polling:polling_load_balance", "//trpc/naming/common/util/loadbalance/weighted_round_robin:weighted_round_robin_load_balancer", ], ) diff --git a/trpc/naming/common/util/loadbalance/hash/BUILD b/trpc/naming/common/util/loadbalance/hash/BUILD new file mode 100644 index 00000000..18fe4760 --- /dev/null +++ b/trpc/naming/common/util/loadbalance/hash/BUILD @@ -0,0 +1,55 @@ +# Description: trpc-cpp. + +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "consistenthash_load_balance", + srcs = ["consistenthash_load_balance.cc"], + hdrs = ["consistenthash_load_balance.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + "//trpc/common/config:loadbalance_naming_conf", + "//trpc/common/config:loadbalance_naming_conf_parser", + "//trpc/common/config:trpc_config", + "//trpc/naming:load_balance_factory", + "//trpc/naming/common/util/hash:hash_func", + "//trpc/naming/common/util/loadbalance/hash:common", + "//trpc/util/log:logging", + ], +) + +cc_library( + name = "modulohash_load_balance", + srcs = ["modulohash_load_balance.cc"], + hdrs = ["modulohash_load_balance.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + "//trpc/common/config:loadbalance_naming_conf", + "//trpc/common/config:loadbalance_naming_conf_parser", + "//trpc/common/config:trpc_config", + "//trpc/naming:load_balance_factory", + "//trpc/naming/common/util/hash:hash_func", + "//trpc/naming/common/util/loadbalance/hash:common", + "//trpc/util/log:logging", + ], +) + +cc_library( + name = "common", + srcs = ["common.cc"], + hdrs = ["common.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + "//trpc/common/config:loadbalance_naming_conf", + "//trpc/naming/common:common_defs", + "//trpc/naming/common/util/hash:hash_func", + ], +) diff --git a/trpc/naming/common/util/loadbalance/hash/common.cc b/trpc/naming/common/util/loadbalance/hash/common.cc new file mode 100644 index 00000000..1da733cf --- /dev/null +++ b/trpc/naming/common/util/loadbalance/hash/common.cc @@ -0,0 +1,111 @@ +// +// +// Tencent is pleased to support the open source community by making tRPC available. +// +// Copyright (C) 2024 THL A29 Limited, a Tencent company. +// All rights reserved. +// +// If you have downloaded a copy of the tRPC source code from Tencent, +// please note that tRPC source code is licensed under the Apache 2.0 License, +// A copy of the Apache 2.0 License is included in this file. +// +// + +#include "trpc/naming/common/util/loadbalance/hash/common.h" + +#include +#include + +#include "trpc/naming/common/common_defs.h" +#include "trpc/naming/common/util/hash/hash_func.h" + +namespace trpc { +std::string GenerateKeysAsString(const SelectorInfo* info, std::vector& indexs) { + std::string key; + for (int index : indexs) { + switch (index) { + case 0: + if (info->context != nullptr) { + key += info->context->GetCallerName(); + } + break; + case 1: + if (info->context != nullptr) { + key += info->context->GetIp(); + } + break; + case 2: + if (info->context != nullptr) { + key += std::to_string(info->context->GetPort()); + } + break; + case 3: + key += info->name; + break; + + case 4: + if (info->context != nullptr) { + key += info->context->GetCalleeName(); + } + break; + case 5: + key += info->load_balance_name; + break; + default: + break; + } + } + return key; +} + +bool CheckLoadBalanceSelectorConfig(naming::LoadBalanceConfig& loadbalance_config_) { + bool res = true; + for (int index : loadbalance_config_.hash_args) { + if (index < 0 || index > kHashNodesMaxIndex) { + TRPC_FMT_DEBUG("index in yaml configuration out of range, use default config"); + res = false; + break; + } + } + if (!res) { + // set to default value + loadbalance_config_.hash_args.assign({0}); + } + if (kHashFuncTable.find(loadbalance_config_.hash_func) == kHashFuncTable.end()) { + res = false; + TRPC_FMT_DEBUG("hash func name is invalid, use default config"); + // set to default value + loadbalance_config_.hash_func = "murmur3"; + } + if (loadbalance_config_.hash_nodes < 1) { + res = false; + TRPC_FMT_DEBUG("hash nodes is invalid, use default config"); + // set to default value + loadbalance_config_.hash_nodes = 20; + } + + return res; +} + +bool CheckLoadbalanceInfoDiff(const std::vector& orig_endpoints, + const std::vector* new_endpoints) { + if (orig_endpoints.size() != new_endpoints->size()) { + return true; + } + + int i = 0; + for (auto& var : *new_endpoints) { + auto orig_endpoint = orig_endpoints[i++]; + if (orig_endpoint.host != var.host || orig_endpoint.port != var.port) { + return true; + } + + if (orig_endpoint.status != var.status) { + return true; + } + } + + return false; +} + +} // namespace trpc \ No newline at end of file diff --git a/trpc/naming/common/util/loadbalance/hash/common.h b/trpc/naming/common/util/loadbalance/hash/common.h new file mode 100644 index 00000000..62661856 --- /dev/null +++ b/trpc/naming/common/util/loadbalance/hash/common.h @@ -0,0 +1,31 @@ +// +// +// Tencent is pleased to support the open source community by making tRPC available. +// +// Copyright (C) 2024 THL A29 Limited, a Tencent company. +// All rights reserved. +// +// If you have downloaded a copy of the tRPC source code from Tencent, +// please note that tRPC source code is licensed under the Apache 2.0 License, +// A copy of the Apache 2.0 License is included in this file. +// +// + +#pragma once + +#include +#include + +#include "trpc/common/config/loadbalance_naming_conf.h" +#include "trpc/naming/common/common_defs.h" +#include "trpc/naming/common/util/hash/hash_func.h" + +namespace trpc { + +std::string GenerateKeysAsString(const SelectorInfo* info, std::vector& indexs); + +bool CheckLoadBalanceSelectorConfig(naming::LoadBalanceConfig& loadbalance_config_); + +bool CheckLoadbalanceInfoDiff(const std::vector& orig_endpoints, + const std::vector* new_endpoints); +} // namespace trpc \ No newline at end of file diff --git a/trpc/naming/common/util/loadbalance/hash/consistenthash_load_balance.cc b/trpc/naming/common/util/loadbalance/hash/consistenthash_load_balance.cc new file mode 100644 index 00000000..c125fb22 --- /dev/null +++ b/trpc/naming/common/util/loadbalance/hash/consistenthash_load_balance.cc @@ -0,0 +1,155 @@ +// +// +// Tencent is pleased to support the open source community by making tRPC available. +// +// Copyright (C) 2024 THL A29 Limited, a Tencent company. +// All rights reserved. +// +// If you have downloaded a copy of the tRPC source code from Tencent, +// please note that tRPC source code is licensed under the Apache 2.0 License, +// A copy of the Apache 2.0 License is included in this file. +// +// + +#include "trpc/naming/common/util/loadbalance/hash/consistenthash_load_balance.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "trpc/common/config/trpc_config.h" +#include "trpc/naming/common/util/hash/hash_func.h" +#include "trpc/naming/common/util/loadbalance/hash/common.h" +#include "trpc/naming/load_balance_factory.h" +#include "trpc/util/log/logging.h" + +namespace trpc { + +int ConsistentHashLoadBalance::Init() noexcept { + if (!trpc::TrpcConfig::GetInstance()->GetPluginConfig("loadbalance", kConsistentHashLoadBalance, + loadbalance_config_)) { + TRPC_FMT_DEBUG("get loadbalance config failed, use default value"); + } + + bool res = CheckLoadBalanceSelectorConfig(loadbalance_config_); + return res ? 0 : -1; +} + +bool ConsistentHashLoadBalance::IsLoadBalanceInfoDiff(const LoadBalanceInfo* info) { + if (nullptr == info || nullptr == info->info || nullptr == info->endpoints) { + return false; + } + + const SelectorInfo* select_info = info->info; + std::shared_lock lock(mutex_); + auto iter = callee_router_infos_.find(select_info->name); + if (callee_router_infos_.end() == iter) { + return true; + } + + return CheckLoadbalanceInfoDiff(callee_router_infos_[select_info->name].endpoints, info->endpoints); +} + +// Update the routing nodes used for load balancing +int ConsistentHashLoadBalance::Update(const LoadBalanceInfo* info) { + if (nullptr == info || nullptr == info->info || nullptr == info->endpoints) { + TRPC_LOG_ERROR("Endpoint info of name is empty"); + return -1; + } + + const SelectorInfo* select_info = info->info; + + if (IsLoadBalanceInfoDiff(info)) { + InnerEndpointInfos old_info; + { + std::unique_lock lock(mutex_); + if (callee_router_infos_.find(select_info->name) != callee_router_infos_.end()) { + old_info = callee_router_infos_[select_info->name]; + } + } + InnerEndpointInfos endpoint_info; + + endpoint_info.hashring = std::move(old_info.hashring); + endpoint_info.endpoints.assign(info->endpoints->begin(), info->endpoints->end()); + + std::unordered_set old_set; + std::unordered_map new_set; + + for (uint32_t i = 0; i < info->endpoints->size(); i++) { + new_set[info->endpoints->at(i).host + std::to_string(info->endpoints->at(i).port)] = i; + } + + for (const auto& endpoint : old_info.endpoints) { + old_set.insert(endpoint.host + std::to_string(endpoint.port)); + } + + for (const auto& elem : old_set) { + if (new_set.find(elem) == new_set.end()) { + for (uint32_t i = 0; i < loadbalance_config_.hash_nodes; i++) { + uint64_t key = Hash(elem + std::to_string(i), loadbalance_config_.hash_func); + endpoint_info.hashring.erase(key); + } + } + } + + for (const auto& elem : new_set) { + if (old_set.find(elem.first) == old_set.end()) { + for (uint32_t i = 0; i < loadbalance_config_.hash_nodes; i++) { + uint64_t key = Hash(elem.first + std::to_string(i), loadbalance_config_.hash_func); + endpoint_info.hashring[key] = info->endpoints->at(elem.second); + } + } + } + + std::unique_lock lock(mutex_); + callee_router_infos_[select_info->name] = endpoint_info; + } + + return 0; +} + +int ConsistentHashLoadBalance::Next(LoadBalanceResult& result) { + if (nullptr == result.info) { + return -1; + } + + std::shared_lock lock(mutex_); + auto iter = callee_router_infos_.find((result.info)->name); + if (iter == callee_router_infos_.end()) { + TRPC_LOG_ERROR("Router info of name " << (result.info)->name << " no found"); + return -1; + } + + std::map& hashring = iter->second.hashring; + size_t endpoints_num = hashring.size(); + if (endpoints_num < 1) { + TRPC_LOG_ERROR("Router info of name is empty"); + return -1; + } + uint64_t hash; + if (result.info->context != nullptr && !result.info->context->GetHashKey().empty()) { + hash = std::stoull(result.info->context->GetHashKey()); + } else { + hash = Hash(GenerateKeysAsString(result.info, loadbalance_config_.hash_args), loadbalance_config_.hash_func); + } + auto info_iter = hashring.lower_bound(hash); + + if (info_iter == hashring.end()) { + info_iter = hashring.begin(); + } + + result.result = info_iter->second; + + return 0; +} + +} // namespace trpc \ No newline at end of file diff --git a/trpc/naming/common/util/loadbalance/hash/consistenthash_load_balance.h b/trpc/naming/common/util/loadbalance/hash/consistenthash_load_balance.h new file mode 100644 index 00000000..b0fc7bc1 --- /dev/null +++ b/trpc/naming/common/util/loadbalance/hash/consistenthash_load_balance.h @@ -0,0 +1,67 @@ +// +// +// Tencent is pleased to support the open source community by making tRPC available. +// +// Copyright (C) 2024 THL A29 Limited, a Tencent company. +// All rights reserved. +// +// If you have downloaded a copy of the tRPC source code from Tencent, +// please note that tRPC source code is licensed under the Apache 2.0 License, +// A copy of the Apache 2.0 License is included in this file. +// +// + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "trpc/common/config/loadbalance_naming_conf.h" +#include "trpc/common/config/loadbalance_naming_conf_parser.h" +#include "trpc/naming/load_balance.h" + +namespace trpc { +constexpr char kConsistentHashLoadBalance[] = "consistent_hash"; + +/// @brief consistent hash load balancing plugin +class ConsistentHashLoadBalance : public LoadBalance { + public: + ConsistentHashLoadBalance() = default; + ~ConsistentHashLoadBalance() = default; + + /// @brief Get the name of the load balancing plugin + std::string Name() const override { return kConsistentHashLoadBalance; } + + /// @brief Initialization + /// @return Returns 0 on success, -1 on failure + int Init() noexcept override; + + /// @brief Update the routing node information used by the load balancing + int Update(const LoadBalanceInfo* info) override; + + /// @brief Return a callee node + int Next(LoadBalanceResult& result) override; + + private: + /// @brief Check if the load balancing information is different + + bool IsLoadBalanceInfoDiff(const LoadBalanceInfo* info); + + struct InnerEndpointInfos { + std::vector endpoints; + std::map hashring; + }; + + std::unordered_map callee_router_infos_; + + naming::LoadBalanceConfig loadbalance_config_; + + std::shared_mutex mutex_; +}; + +} // namespace trpc diff --git a/trpc/naming/common/util/loadbalance/hash/modulohash_load_balance.cc b/trpc/naming/common/util/loadbalance/hash/modulohash_load_balance.cc new file mode 100644 index 00000000..8cb1a273 --- /dev/null +++ b/trpc/naming/common/util/loadbalance/hash/modulohash_load_balance.cc @@ -0,0 +1,109 @@ +// +// +// Tencent is pleased to support the open source community by making tRPC available. +// +// Copyright (C) 2024 THL A29 Limited, a Tencent company. +// All rights reserved. +// +// If you have downloaded a copy of the tRPC source code from Tencent, +// please note that tRPC source code is licensed under the Apache 2.0 License, +// A copy of the Apache 2.0 License is included in this file. +// +// + +#include "trpc/naming/common/util/loadbalance/hash/modulohash_load_balance.h" + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "trpc/common/config/trpc_config.h" +#include "trpc/naming/common/common_defs.h" +#include "trpc/naming/common/util/hash/hash_func.h" +#include "trpc/naming/common/util/loadbalance/hash/common.h" +#include "trpc/naming/load_balance_factory.h" +#include "trpc/util/log/logging.h" + +namespace trpc { + +int ModuloHashLoadBalance::Init() noexcept { + if (!trpc::TrpcConfig::GetInstance()->GetPluginConfig("loadbalance", kModuloHashLoadBalance, loadbalance_config_)) { + TRPC_FMT_DEBUG("get loadbalance config failed, use default value"); + } + + bool res = CheckLoadBalanceSelectorConfig(loadbalance_config_); + return res ? 0 : -1; +} + +bool ModuloHashLoadBalance::IsLoadBalanceInfoDiff(const LoadBalanceInfo* info) { + if (nullptr == info || nullptr == info->info || nullptr == info->endpoints) { + return false; + } + + const SelectorInfo* select_info = info->info; + std::shared_lock lock(mutex_); + if (callee_router_infos_.end() == callee_router_infos_.find(select_info->name)) { + return true; + } + + return CheckLoadbalanceInfoDiff(callee_router_infos_[select_info->name], info->endpoints); +} + +// Update the routing nodes used for load balancing +int ModuloHashLoadBalance::Update(const LoadBalanceInfo* info) { + if (nullptr == info || nullptr == info->info || nullptr == info->endpoints) { + TRPC_LOG_ERROR("Endpoint info of name is empty"); + return -1; + } + + const SelectorInfo* select_info = info->info; + + if (IsLoadBalanceInfoDiff(info)) { + std::vector endpoints; + endpoints.assign(info->endpoints->begin(), info->endpoints->end()); + + std::unique_lock lock(mutex_); + callee_router_infos_[select_info->name] = endpoints; + } + + return 0; +} + +int ModuloHashLoadBalance::Next(LoadBalanceResult& result) { + if (nullptr == result.info) { + return -1; + } + + std::shared_lock lock(mutex_); + auto iter = callee_router_infos_.find((result.info)->name); + if (iter == callee_router_infos_.end()) { + TRPC_LOG_ERROR("Router info of name " << (result.info)->name << " no found"); + return -1; + } + + std::vector& endpoints = iter->second; + size_t endpoints_num = endpoints.size(); + if (endpoints_num < 1) { + TRPC_LOG_ERROR("Router info of name is empty"); + return -1; + } + + uint64_t hash; + if (result.info->context != nullptr && !result.info->context->GetHashKey().empty()) { + hash = std::stoull(result.info->context->GetHashKey()) % endpoints_num; + } else { + hash = Hash(GenerateKeysAsString(result.info, loadbalance_config_.hash_args), loadbalance_config_.hash_func, + endpoints_num); + } + result.result = endpoints[hash]; + + return 0; +} + +} // namespace trpc \ No newline at end of file diff --git a/trpc/naming/common/util/loadbalance/hash/modulohash_load_balance.h b/trpc/naming/common/util/loadbalance/hash/modulohash_load_balance.h new file mode 100644 index 00000000..932a9636 --- /dev/null +++ b/trpc/naming/common/util/loadbalance/hash/modulohash_load_balance.h @@ -0,0 +1,61 @@ +// +// +// Tencent is pleased to support the open source community by making tRPC available. +// +// Copyright (C) 2024 THL A29 Limited, a Tencent company. +// All rights reserved. +// +// If you have downloaded a copy of the tRPC source code from Tencent, +// please note that tRPC source code is licensed under the Apache 2.0 License, +// A copy of the Apache 2.0 License is included in this file. +// +// + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "trpc/common/config/loadbalance_naming_conf.h" +#include "trpc/common/config/loadbalance_naming_conf_parser.h" +#include "trpc/naming/common/common_defs.h" +#include "trpc/naming/load_balance.h" + +namespace trpc { +constexpr char kModuloHashLoadBalance[] = "modulo_hash"; + +/// @brief consistent hash load balancing plugin +class ModuloHashLoadBalance : public LoadBalance { + public: + ModuloHashLoadBalance() = default; + ~ModuloHashLoadBalance() = default; + + /// @brief Get the name of the load balancing plugin + std::string Name() const override { return kModuloHashLoadBalance; } + + /// @brief Initialization + /// @return Returns 0 on success, -1 on failure + int Init() noexcept override; + + /// @brief Update the routing node information used by the load balancing + int Update(const LoadBalanceInfo* info) override; + + /// @brief Return a callee node + int Next(LoadBalanceResult& result) override; + + private: + /// @brief Check if the load balancing information is different + bool IsLoadBalanceInfoDiff(const LoadBalanceInfo* info); + + std::unordered_map> callee_router_infos_; + + naming::LoadBalanceConfig loadbalance_config_; + + std::shared_mutex mutex_; +}; + +} // namespace trpc diff --git a/trpc/naming/common/util/loadbalance/trpc_load_balance.cc b/trpc/naming/common/util/loadbalance/trpc_load_balance.cc index f56e6c35..926be93c 100644 --- a/trpc/naming/common/util/loadbalance/trpc_load_balance.cc +++ b/trpc/naming/common/util/loadbalance/trpc_load_balance.cc @@ -10,23 +10,68 @@ // A copy of the Apache 2.0 License is included in this file. // // + +#include "trpc/naming/common/util/loadbalance/trpc_load_balance.h" + +#include "trpc/naming/common/util/loadbalance/hash/consistenthash_load_balance.h" +#include "trpc/naming/common/util/loadbalance/hash/modulohash_load_balance.h" +#include "trpc/naming/common/util/loadbalance/polling/polling_load_balance.h" #include "trpc/naming/common/util/loadbalance/weighted_round_robin/weighted_round_robin_load_balancer.h" #include "trpc/naming/load_balance_factory.h" + namespace trpc::loadbalance { bool Init() { + bool res = true; + LoadBalancePtr swround_robin_load_balance = trpc::LoadBalanceFactory::GetInstance()->Get(kSWRoundRobinLoadBalance); if (swround_robin_load_balance == nullptr) { // Register the default load balancer swround_robin_load_balance = MakeRefCounted(); LoadBalanceFactory::GetInstance()->Register(swround_robin_load_balance); } - return true; + + // Registers default loadbalance plugins which provided by the framework. + LoadBalancePtr polling_load_balance = trpc::LoadBalanceFactory::GetInstance()->Get(kPollingLoadBalance); + if (polling_load_balance == nullptr) { + polling_load_balance = MakeRefCounted(); + int ret = polling_load_balance->Init(); + + if (!ret) { + LoadBalanceFactory::GetInstance()->Register(polling_load_balance); + } else { + res = false; + } + } + + LoadBalancePtr consistenthash_load_balance = trpc::LoadBalanceFactory::GetInstance()->Get(kConsistentHashLoadBalance); + if (consistenthash_load_balance == nullptr) { + consistenthash_load_balance = MakeRefCounted(); + int ret = consistenthash_load_balance->Init(); + if (!ret) { + LoadBalanceFactory::GetInstance()->Register(consistenthash_load_balance); + } else { + res = false; + } + } + + LoadBalancePtr modulohash_load_balance = trpc::LoadBalanceFactory::GetInstance()->Get(kModuloHashLoadBalance); + if (modulohash_load_balance == nullptr) { + modulohash_load_balance = MakeRefCounted(); + int ret = modulohash_load_balance->Init(); + if (!ret) { + LoadBalanceFactory::GetInstance()->Register(modulohash_load_balance); + } else { + res = false; + } + } + + return res; } void Stop() {} void Destroy() { LoadBalanceFactory::GetInstance()->Clear(); } -} // namespace trpc::loadbalance \ No newline at end of file +} // namespace trpc::loadbalance