From 66c888d3ad2a94b918984d7735e7bf5f70b78840 Mon Sep 17 00:00:00 2001 From: Vadim Averin Date: Thu, 21 Dec 2023 07:09:21 +0000 Subject: [PATCH 1/9] Apply patch with changes --- .../yql/udfs/common/ip_base/lib/ip_base_udf.h | 133 ++++++++++-------- .../yql/udfs/common/ip_base/lib/ya.make | 1 + 2 files changed, 79 insertions(+), 55 deletions(-) diff --git a/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h b/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h index 95c98006974f..203967cea53d 100644 --- a/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h +++ b/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h @@ -2,7 +2,7 @@ #include -#include +#include #include namespace { @@ -13,39 +13,76 @@ namespace { using TUnboxedValue = NKikimr::NUdf::TUnboxedValue; using TUnboxedValuePod = NKikimr::NUdf::TUnboxedValuePod; - struct TSerializeIpVisitor { - TStringRef operator()(const TIp4& ip) const { - return TStringRef(reinterpret_cast(&ip), 4); - } - TStringRef operator()(const TIp6& ip) const { - return TStringRef(reinterpret_cast(&ip.Data), 16); - } + struct TRawIp4 { + ui8 a, b, c, d; }; - SIMPLE_STRICT_UDF(TFromString, TOptionalString(TAutoMapString)) { - try { - TString input(args[0].AsStringRef()); - const TIp4Or6& ip = Ip4Or6FromString(input.c_str()); - return valueBuilder->NewString(std::visit(TSerializeIpVisitor(), ip)); - } catch (TSystemError&) { - return TUnboxedValue(); + struct TRawIp6 { + ui8 a1, a0, b1, b0, c1, c0, d1, d0, e1, e0, f1, f0, g1, g0, h1, h0; + }; + + TIpv6Address DeserializeAddress(const TStringRef& str) { + TIpv6Address addr; + if (str.Size() == 4) { + TRawIp4 addr4; + memcpy(&addr4, str.Data(), sizeof addr4); + addr = {addr4.a, addr4.b, addr4.c, addr4.d}; + } else if (str.Size() == 16) { + TRawIp6 addr6; + memcpy(&addr6, str.Data(), sizeof addr6); + addr = {ui16(ui32(addr6.a1) << ui32(8) | ui32(addr6.a0)), + ui16(ui32(addr6.b1) << ui32(8) | ui32(addr6.b0)), + ui16(ui32(addr6.c1) << ui32(8) | ui32(addr6.c0)), + ui16(ui32(addr6.d1) << ui32(8) | ui32(addr6.d0)), + ui16(ui32(addr6.e1) << ui32(8) | ui32(addr6.e0)), + ui16(ui32(addr6.f1) << ui32(8) | ui32(addr6.f0)), + ui16(ui32(addr6.g1) << ui32(8) | ui32(addr6.g0)), + ui16(ui32(addr6.h1) << ui32(8) | ui32(addr6.h0)), + }; + } else { + ythrow yexception() << "Incorrect size of input, expected " + << "4 or 16, got " << str.Size(); } + return addr; } - SIMPLE_UDF(TToString, char*(TAutoMapString)) { - const auto& ref = args[0].AsStringRef(); - if (ref.Size() == 4) { - TIp4 ip; - memcpy(&ip, ref.Data(), sizeof(ip)); - return valueBuilder->NewString(Ip4Or6ToString(ip)); - } else if (ref.Size() == 16) { - TIp6 ip; - memcpy(&ip.Data, ref.Data(), sizeof(ip.Data)); - return valueBuilder->NewString(Ip4Or6ToString(ip)); + TMaybe SerializeAddress(const TIpv6Address& addr) { + TString res; + ui128 x = addr; + if (addr.Type() == TIpv6Address::Ipv4) { + TRawIp4 addr4 { + ui8(x >> 24 & 0xff), + ui8(x >> 16 & 0xff), + ui8(x >> 8 & 0xff), + ui8(x & 0xff) + }; + res = TString(reinterpret_cast(&addr4), sizeof addr4); + } else if (addr.Type() == TIpv6Address::Ipv6) { + TRawIp6 addr6 { + ui8(x >> 120 & 0xff), ui8(x >> 112 & 0xff), + ui8(x >> 104 & 0xff), ui8(x >> 96 & 0xff), + ui8(x >> 88 & 0xff), ui8(x >> 80 & 0xff), + ui8(x >> 72 & 0xff), ui8(x >> 64 & 0xff), + ui8(x >> 56 & 0xff), ui8(x >> 48 & 0xff), + ui8(x >> 40 & 0xff), ui8(x >> 32 & 0xff), + ui8(x >> 24 & 0xff), ui8(x >> 16 & 0xff), + ui8(x >> 8 & 0xff), ui8(x & 0xff) + }; + res = TString(reinterpret_cast(&addr6), sizeof addr6); } else { - ythrow yexception() << "Incorrect size of input, expected " - << "4 or 16, got " << ref.Size(); + return Nothing(); } + return res; + } + + SIMPLE_STRICT_UDF(TFromString, TOptionalString(TAutoMapString)) { + TIpv6Address addr = TIpv6Address::FromString(args[0].AsStringRef()); + auto res = SerializeAddress(addr); + return res ? valueBuilder->NewString(res.GetRef()) : TUnboxedValue(TUnboxedValuePod()); + } + + SIMPLE_UDF(TToString, char*(TAutoMapString)) { + return valueBuilder->NewString(DeserializeAddress(args[0].AsStringRef()).ToString(false)); } SIMPLE_STRICT_UDF(TIsIPv4, bool(TOptionalString)) { @@ -73,15 +110,8 @@ namespace { bool result = false; if (args[0]) { const auto ref = args[0].AsStringRef(); - if (ref.Size() == 16 && ref.Data()[10] == -1) { - bool allZeroes = true; - for (int i = 0; i < 10; ++i) { - if (ref.Data()[i] != 0) { - allZeroes = false; - break; - } - } - result = allZeroes; + if (ref.Size() == 16) { + result = DeserializeAddress(ref).Isv4MappedTov6(); } } return TUnboxedValuePod(result); @@ -92,10 +122,8 @@ namespace { if (ref.Size() == 16) { return valueBuilder->NewString(ref); } else if (ref.Size() == 4) { - TIp4 ipv4; - memcpy(&ipv4, ref.Data(), sizeof(ipv4)); - const TIp6 ipv6 = Ip6FromIp4(ipv4); - return valueBuilder->NewString(TStringRef(reinterpret_cast(&ipv6.Data), 16)); + auto addr6 = TIpv6Address::FromString("::ffff:" + DeserializeAddress(ref).ToString(false)); + return valueBuilder->NewString(SerializeAddress(addr6).GetRef()); } else { ythrow yexception() << "Incorrect size of input, expected " << "4 or 16, got " << ref.Size(); @@ -105,33 +133,28 @@ namespace { SIMPLE_UDF_WITH_OPTIONAL_ARGS(TGetSubnet, char*(TAutoMapString, TOptionalByte), 1) { const auto ref = args[0].AsStringRef(); ui8 subnetSize = args[1].GetOrDefault(0); - + TIpv6Address addr = DeserializeAddress(ref); if (ref.Size() == 4) { if (!subnetSize) { subnetSize = 24; } + if (subnetSize > 32) { + subnetSize = 32; + } } else if (ref.Size() == 16) { if (!subnetSize) { subnetSize = 64; } + if (subnetSize > 128) { + subnetSize = 128; + } } else { ythrow yexception() << "Incorrect size of input, expected " << "4 or 16, got " << ref.Size(); } - TBuffer result(ref.Data(), ref.Size()); - int bytesToMask = ref.Size() * 8 - subnetSize; - ui8 currentByte = ref.Size() - 1; - while (bytesToMask > 0) { - if (bytesToMask > 8) { - result.Data()[currentByte] = 0; - } else { - result.Data()[currentByte] = result.Data()[currentByte] & (0xff << bytesToMask); - } - bytesToMask -= 8; - --currentByte; - } - - return valueBuilder->NewString(TStringRef(result.Data(), result.Size())); + ui128 mask = ui128(-1) << int((addr.Type() == TIpv6Address::Ipv6 ? 128 : 32) - subnetSize); + TIpv6Address beg(ui128(addr) & mask, addr.Type()); + return valueBuilder->NewString(SerializeAddress(beg).GetRef()); } #define EXPORTED_IP_BASE_UDF \ diff --git a/ydb/library/yql/udfs/common/ip_base/lib/ya.make b/ydb/library/yql/udfs/common/ip_base/lib/ya.make index 2d532234dd62..1fc0c6b9a51d 100644 --- a/ydb/library/yql/udfs/common/ip_base/lib/ya.make +++ b/ydb/library/yql/udfs/common/ip_base/lib/ya.make @@ -12,6 +12,7 @@ SRCS( PEERDIR( ydb/library/yql/public/udf + library/cpp/ipmath ) END() From 4ee28583d2199b55f32cf20b9ec36b358cbf6462 Mon Sep 17 00:00:00 2001 From: Vadim Averin Date: Wed, 27 Dec 2023 15:02:44 +0000 Subject: [PATCH 2/9] Add & canonize some tests --- .../canondata/test.test_Basic_/results.txt | 93 +++++++++++++++++++ .../udfs/common/ip_base/test/cases/Basic.in | 1 + .../udfs/common/ip_base/test/cases/Basic.sql | 4 +- 3 files changed, 97 insertions(+), 1 deletion(-) diff --git a/ydb/library/yql/udfs/common/ip_base/test/canondata/test.test_Basic_/results.txt b/ydb/library/yql/udfs/common/ip_base/test/canondata/test.test_Basic_/results.txt index c80bc2d49858..c62c9cbd35f5 100644 --- a/ydb/library/yql/udfs/common/ip_base/test/canondata/test.test_Basic_/results.txt +++ b/ydb/library/yql/udfs/common/ip_base/test/canondata/test.test_Basic_/results.txt @@ -87,6 +87,26 @@ "String" ] ] + ]; + [ + "single_subnet4"; + [ + "OptionalType"; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "single_subnet6"; + [ + "OptionalType"; + [ + "DataType"; + "String" + ] + ] ] ] ] @@ -113,6 +133,12 @@ ]; [ "127.0.0.0" + ]; + [ + "127.0.0.1" + ]; + [ + "127.0.0.1" ] ]; [ @@ -136,6 +162,12 @@ ]; [ "::" + ]; + [ + "::" + ]; + [ + "::1" ] ]; [ @@ -161,6 +193,12 @@ ]; [ "213.180.0.0" + ]; + [ + "213.180.193.3" + ]; + [ + "213.180.193.3" ] ]; [ @@ -186,6 +224,12 @@ ]; [ "2a02::" + ]; + [ + "2a02:6b8::" + ]; + [ + "2a02:6b8::3" ] ]; [ @@ -211,6 +255,12 @@ ]; [ "2400::" + ]; + [ + "2400:cb00::" + ]; + [ + "2400:cb00:2048:1::681c:1b65" ] ]; [ @@ -236,6 +286,12 @@ ]; [ "fe80::" + ]; + [ + "fe80::" + ]; + [ + "fe80::215:b2ff:fea9:67ce" ] ]; [ @@ -261,6 +317,12 @@ ]; [ "::" + ]; + [ + "::" + ]; + [ + "::ffff:77.75.155.3" ] ]; [ @@ -272,7 +334,38 @@ #; #; #; + #; + #; # + ]; + [ + [ + "\0\0\0\0" + ]; + [ + "0.0.0.0" + ]; + %true; + %false; + %false; + [ + "::ffff:0.0.0.0" + ]; + [ + "0.0.0.0" + ]; + [ + "0.0.0.0" + ]; + [ + "0.0.0.0" + ]; + [ + "0.0.0.0" + ]; + [ + "0.0.0.0" + ] ] ] } diff --git a/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.in b/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.in index 8e7fa7925852..4aa205991410 100644 --- a/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.in +++ b/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.in @@ -6,3 +6,4 @@ {"key"="fe80::215:b2ff:fea9:67ce";"subkey"="";"value"=""}; {"key"="::ffff:77.75.155.3";"subkey"="";"value"=""}; {"key"="sdfsdfsdf";"subkey"="";"value"=""}; +{"key"="0.0.0.0";"subkey"="";value=""}; \ No newline at end of file diff --git a/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.sql b/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.sql index effb59e5ee7f..1b875bc7313a 100644 --- a/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.sql +++ b/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.sql @@ -8,7 +8,9 @@ SELECT Ip::ToString(Ip::ConvertToIPv6(internal_representation)) AS all_ipv6, Ip::ToString(Ip::GetSubnet(internal_representation)) AS default_subnet, Ip::ToString(Ip::GetSubnet(internal_representation, 125)) AS small_subnet, - Ip::ToString(Ip::GetSubnet(internal_representation, 16)) AS large_subnet + Ip::ToString(Ip::GetSubnet(internal_representation, 16)) AS large_subnet, + Ip::ToString(Ip::GetSubnet(internal_representation, 32)) AS single_subnet4, + Ip::ToString(Ip::GetSubnet(internal_representation, 128)) AS single_subnet6 FROM ( SELECT Ip::FromString(key) AS internal_representation FROM Input ); From 15c6cfd64aa5d45c978a4cbd1aa173c978506bdd Mon Sep 17 00:00:00 2001 From: Vadim Averin Date: Wed, 17 Jan 2024 13:07:18 +0000 Subject: [PATCH 3/9] More refactoring --- ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h b/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h index 203967cea53d..210a01694f37 100644 --- a/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h +++ b/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h @@ -78,7 +78,7 @@ namespace { SIMPLE_STRICT_UDF(TFromString, TOptionalString(TAutoMapString)) { TIpv6Address addr = TIpv6Address::FromString(args[0].AsStringRef()); auto res = SerializeAddress(addr); - return res ? valueBuilder->NewString(res.GetRef()) : TUnboxedValue(TUnboxedValuePod()); + return res ? valueBuilder->NewString(res.GetRef()) : TUnboxedValue(); } SIMPLE_UDF(TToString, char*(TAutoMapString)) { @@ -122,7 +122,8 @@ namespace { if (ref.Size() == 16) { return valueBuilder->NewString(ref); } else if (ref.Size() == 4) { - auto addr6 = TIpv6Address::FromString("::ffff:" + DeserializeAddress(ref).ToString(false)); + TIpv6Address addr4 = DeserializeAddress(ref); + auto addr6 = TIpv6Address(ui128(addr4) | ui128(0xFFFF) << 32, TIpv6Address::Ipv6); return valueBuilder->NewString(SerializeAddress(addr6).GetRef()); } else { ythrow yexception() << "Incorrect size of input, expected " @@ -152,8 +153,7 @@ namespace { ythrow yexception() << "Incorrect size of input, expected " << "4 or 16, got " << ref.Size(); } - ui128 mask = ui128(-1) << int((addr.Type() == TIpv6Address::Ipv6 ? 128 : 32) - subnetSize); - TIpv6Address beg(ui128(addr) & mask, addr.Type()); + TIpv6Address beg = LowerBoundForPrefix(addr, subnetSize); return valueBuilder->NewString(SerializeAddress(beg).GetRef()); } From f39371edb4c97053e45b7307e316811583f808df Mon Sep 17 00:00:00 2001 From: Vadim Averin Date: Wed, 17 Jan 2024 13:35:24 +0000 Subject: [PATCH 4/9] Final cleanup --- .../yql/udfs/common/ip_base/lib/ip_base_udf.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h b/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h index 210a01694f37..4349cff216f2 100644 --- a/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h +++ b/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h @@ -46,7 +46,8 @@ namespace { return addr; } - TMaybe SerializeAddress(const TIpv6Address& addr) { + TString SerializeAddress(const TIpv6Address& addr) { + Y_ENSURE(addr.Type() == TIpv6Address::Ipv4 || addr.Type() == TIpv6Address::Ipv6); TString res; ui128 x = addr; if (addr.Type() == TIpv6Address::Ipv4) { @@ -69,16 +70,16 @@ namespace { ui8(x >> 8 & 0xff), ui8(x & 0xff) }; res = TString(reinterpret_cast(&addr6), sizeof addr6); - } else { - return Nothing(); } return res; } SIMPLE_STRICT_UDF(TFromString, TOptionalString(TAutoMapString)) { TIpv6Address addr = TIpv6Address::FromString(args[0].AsStringRef()); - auto res = SerializeAddress(addr); - return res ? valueBuilder->NewString(res.GetRef()) : TUnboxedValue(); + if (addr.Type() != TIpv6Address::Ipv4 && addr.Type() != TIpv6Address::Ipv6) { + return TUnboxedValue(); + } + return valueBuilder->NewString(SerializeAddress(addr)); } SIMPLE_UDF(TToString, char*(TAutoMapString)) { @@ -124,7 +125,7 @@ namespace { } else if (ref.Size() == 4) { TIpv6Address addr4 = DeserializeAddress(ref); auto addr6 = TIpv6Address(ui128(addr4) | ui128(0xFFFF) << 32, TIpv6Address::Ipv6); - return valueBuilder->NewString(SerializeAddress(addr6).GetRef()); + return valueBuilder->NewString(SerializeAddress(addr6)); } else { ythrow yexception() << "Incorrect size of input, expected " << "4 or 16, got " << ref.Size(); @@ -154,7 +155,7 @@ namespace { << "4 or 16, got " << ref.Size(); } TIpv6Address beg = LowerBoundForPrefix(addr, subnetSize); - return valueBuilder->NewString(SerializeAddress(beg).GetRef()); + return valueBuilder->NewString(SerializeAddress(beg)); } #define EXPORTED_IP_BASE_UDF \ From 0c3ba9edb97efbbf15fb44aec91a4d1b828289ce Mon Sep 17 00:00:00 2001 From: Vadim Averin Date: Thu, 21 Dec 2023 07:09:21 +0000 Subject: [PATCH 5/9] Apply patch with changes --- .../yql/udfs/common/ip_base/lib/ip_base_udf.h | 133 ++++++++++-------- .../yql/udfs/common/ip_base/lib/ya.make | 1 + 2 files changed, 79 insertions(+), 55 deletions(-) diff --git a/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h b/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h index 95c98006974f..203967cea53d 100644 --- a/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h +++ b/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h @@ -2,7 +2,7 @@ #include -#include +#include #include namespace { @@ -13,39 +13,76 @@ namespace { using TUnboxedValue = NKikimr::NUdf::TUnboxedValue; using TUnboxedValuePod = NKikimr::NUdf::TUnboxedValuePod; - struct TSerializeIpVisitor { - TStringRef operator()(const TIp4& ip) const { - return TStringRef(reinterpret_cast(&ip), 4); - } - TStringRef operator()(const TIp6& ip) const { - return TStringRef(reinterpret_cast(&ip.Data), 16); - } + struct TRawIp4 { + ui8 a, b, c, d; }; - SIMPLE_STRICT_UDF(TFromString, TOptionalString(TAutoMapString)) { - try { - TString input(args[0].AsStringRef()); - const TIp4Or6& ip = Ip4Or6FromString(input.c_str()); - return valueBuilder->NewString(std::visit(TSerializeIpVisitor(), ip)); - } catch (TSystemError&) { - return TUnboxedValue(); + struct TRawIp6 { + ui8 a1, a0, b1, b0, c1, c0, d1, d0, e1, e0, f1, f0, g1, g0, h1, h0; + }; + + TIpv6Address DeserializeAddress(const TStringRef& str) { + TIpv6Address addr; + if (str.Size() == 4) { + TRawIp4 addr4; + memcpy(&addr4, str.Data(), sizeof addr4); + addr = {addr4.a, addr4.b, addr4.c, addr4.d}; + } else if (str.Size() == 16) { + TRawIp6 addr6; + memcpy(&addr6, str.Data(), sizeof addr6); + addr = {ui16(ui32(addr6.a1) << ui32(8) | ui32(addr6.a0)), + ui16(ui32(addr6.b1) << ui32(8) | ui32(addr6.b0)), + ui16(ui32(addr6.c1) << ui32(8) | ui32(addr6.c0)), + ui16(ui32(addr6.d1) << ui32(8) | ui32(addr6.d0)), + ui16(ui32(addr6.e1) << ui32(8) | ui32(addr6.e0)), + ui16(ui32(addr6.f1) << ui32(8) | ui32(addr6.f0)), + ui16(ui32(addr6.g1) << ui32(8) | ui32(addr6.g0)), + ui16(ui32(addr6.h1) << ui32(8) | ui32(addr6.h0)), + }; + } else { + ythrow yexception() << "Incorrect size of input, expected " + << "4 or 16, got " << str.Size(); } + return addr; } - SIMPLE_UDF(TToString, char*(TAutoMapString)) { - const auto& ref = args[0].AsStringRef(); - if (ref.Size() == 4) { - TIp4 ip; - memcpy(&ip, ref.Data(), sizeof(ip)); - return valueBuilder->NewString(Ip4Or6ToString(ip)); - } else if (ref.Size() == 16) { - TIp6 ip; - memcpy(&ip.Data, ref.Data(), sizeof(ip.Data)); - return valueBuilder->NewString(Ip4Or6ToString(ip)); + TMaybe SerializeAddress(const TIpv6Address& addr) { + TString res; + ui128 x = addr; + if (addr.Type() == TIpv6Address::Ipv4) { + TRawIp4 addr4 { + ui8(x >> 24 & 0xff), + ui8(x >> 16 & 0xff), + ui8(x >> 8 & 0xff), + ui8(x & 0xff) + }; + res = TString(reinterpret_cast(&addr4), sizeof addr4); + } else if (addr.Type() == TIpv6Address::Ipv6) { + TRawIp6 addr6 { + ui8(x >> 120 & 0xff), ui8(x >> 112 & 0xff), + ui8(x >> 104 & 0xff), ui8(x >> 96 & 0xff), + ui8(x >> 88 & 0xff), ui8(x >> 80 & 0xff), + ui8(x >> 72 & 0xff), ui8(x >> 64 & 0xff), + ui8(x >> 56 & 0xff), ui8(x >> 48 & 0xff), + ui8(x >> 40 & 0xff), ui8(x >> 32 & 0xff), + ui8(x >> 24 & 0xff), ui8(x >> 16 & 0xff), + ui8(x >> 8 & 0xff), ui8(x & 0xff) + }; + res = TString(reinterpret_cast(&addr6), sizeof addr6); } else { - ythrow yexception() << "Incorrect size of input, expected " - << "4 or 16, got " << ref.Size(); + return Nothing(); } + return res; + } + + SIMPLE_STRICT_UDF(TFromString, TOptionalString(TAutoMapString)) { + TIpv6Address addr = TIpv6Address::FromString(args[0].AsStringRef()); + auto res = SerializeAddress(addr); + return res ? valueBuilder->NewString(res.GetRef()) : TUnboxedValue(TUnboxedValuePod()); + } + + SIMPLE_UDF(TToString, char*(TAutoMapString)) { + return valueBuilder->NewString(DeserializeAddress(args[0].AsStringRef()).ToString(false)); } SIMPLE_STRICT_UDF(TIsIPv4, bool(TOptionalString)) { @@ -73,15 +110,8 @@ namespace { bool result = false; if (args[0]) { const auto ref = args[0].AsStringRef(); - if (ref.Size() == 16 && ref.Data()[10] == -1) { - bool allZeroes = true; - for (int i = 0; i < 10; ++i) { - if (ref.Data()[i] != 0) { - allZeroes = false; - break; - } - } - result = allZeroes; + if (ref.Size() == 16) { + result = DeserializeAddress(ref).Isv4MappedTov6(); } } return TUnboxedValuePod(result); @@ -92,10 +122,8 @@ namespace { if (ref.Size() == 16) { return valueBuilder->NewString(ref); } else if (ref.Size() == 4) { - TIp4 ipv4; - memcpy(&ipv4, ref.Data(), sizeof(ipv4)); - const TIp6 ipv6 = Ip6FromIp4(ipv4); - return valueBuilder->NewString(TStringRef(reinterpret_cast(&ipv6.Data), 16)); + auto addr6 = TIpv6Address::FromString("::ffff:" + DeserializeAddress(ref).ToString(false)); + return valueBuilder->NewString(SerializeAddress(addr6).GetRef()); } else { ythrow yexception() << "Incorrect size of input, expected " << "4 or 16, got " << ref.Size(); @@ -105,33 +133,28 @@ namespace { SIMPLE_UDF_WITH_OPTIONAL_ARGS(TGetSubnet, char*(TAutoMapString, TOptionalByte), 1) { const auto ref = args[0].AsStringRef(); ui8 subnetSize = args[1].GetOrDefault(0); - + TIpv6Address addr = DeserializeAddress(ref); if (ref.Size() == 4) { if (!subnetSize) { subnetSize = 24; } + if (subnetSize > 32) { + subnetSize = 32; + } } else if (ref.Size() == 16) { if (!subnetSize) { subnetSize = 64; } + if (subnetSize > 128) { + subnetSize = 128; + } } else { ythrow yexception() << "Incorrect size of input, expected " << "4 or 16, got " << ref.Size(); } - TBuffer result(ref.Data(), ref.Size()); - int bytesToMask = ref.Size() * 8 - subnetSize; - ui8 currentByte = ref.Size() - 1; - while (bytesToMask > 0) { - if (bytesToMask > 8) { - result.Data()[currentByte] = 0; - } else { - result.Data()[currentByte] = result.Data()[currentByte] & (0xff << bytesToMask); - } - bytesToMask -= 8; - --currentByte; - } - - return valueBuilder->NewString(TStringRef(result.Data(), result.Size())); + ui128 mask = ui128(-1) << int((addr.Type() == TIpv6Address::Ipv6 ? 128 : 32) - subnetSize); + TIpv6Address beg(ui128(addr) & mask, addr.Type()); + return valueBuilder->NewString(SerializeAddress(beg).GetRef()); } #define EXPORTED_IP_BASE_UDF \ diff --git a/ydb/library/yql/udfs/common/ip_base/lib/ya.make b/ydb/library/yql/udfs/common/ip_base/lib/ya.make index 2d532234dd62..1fc0c6b9a51d 100644 --- a/ydb/library/yql/udfs/common/ip_base/lib/ya.make +++ b/ydb/library/yql/udfs/common/ip_base/lib/ya.make @@ -12,6 +12,7 @@ SRCS( PEERDIR( ydb/library/yql/public/udf + library/cpp/ipmath ) END() From 0136b9cc56532956077e7589c49a744b4bf695da Mon Sep 17 00:00:00 2001 From: Vadim Averin Date: Wed, 27 Dec 2023 15:02:44 +0000 Subject: [PATCH 6/9] Add & canonize some tests --- .../canondata/test.test_Basic_/results.txt | 93 +++++++++++++++++++ .../udfs/common/ip_base/test/cases/Basic.in | 1 + .../udfs/common/ip_base/test/cases/Basic.sql | 4 +- 3 files changed, 97 insertions(+), 1 deletion(-) diff --git a/ydb/library/yql/udfs/common/ip_base/test/canondata/test.test_Basic_/results.txt b/ydb/library/yql/udfs/common/ip_base/test/canondata/test.test_Basic_/results.txt index c80bc2d49858..c62c9cbd35f5 100644 --- a/ydb/library/yql/udfs/common/ip_base/test/canondata/test.test_Basic_/results.txt +++ b/ydb/library/yql/udfs/common/ip_base/test/canondata/test.test_Basic_/results.txt @@ -87,6 +87,26 @@ "String" ] ] + ]; + [ + "single_subnet4"; + [ + "OptionalType"; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "single_subnet6"; + [ + "OptionalType"; + [ + "DataType"; + "String" + ] + ] ] ] ] @@ -113,6 +133,12 @@ ]; [ "127.0.0.0" + ]; + [ + "127.0.0.1" + ]; + [ + "127.0.0.1" ] ]; [ @@ -136,6 +162,12 @@ ]; [ "::" + ]; + [ + "::" + ]; + [ + "::1" ] ]; [ @@ -161,6 +193,12 @@ ]; [ "213.180.0.0" + ]; + [ + "213.180.193.3" + ]; + [ + "213.180.193.3" ] ]; [ @@ -186,6 +224,12 @@ ]; [ "2a02::" + ]; + [ + "2a02:6b8::" + ]; + [ + "2a02:6b8::3" ] ]; [ @@ -211,6 +255,12 @@ ]; [ "2400::" + ]; + [ + "2400:cb00::" + ]; + [ + "2400:cb00:2048:1::681c:1b65" ] ]; [ @@ -236,6 +286,12 @@ ]; [ "fe80::" + ]; + [ + "fe80::" + ]; + [ + "fe80::215:b2ff:fea9:67ce" ] ]; [ @@ -261,6 +317,12 @@ ]; [ "::" + ]; + [ + "::" + ]; + [ + "::ffff:77.75.155.3" ] ]; [ @@ -272,7 +334,38 @@ #; #; #; + #; + #; # + ]; + [ + [ + "\0\0\0\0" + ]; + [ + "0.0.0.0" + ]; + %true; + %false; + %false; + [ + "::ffff:0.0.0.0" + ]; + [ + "0.0.0.0" + ]; + [ + "0.0.0.0" + ]; + [ + "0.0.0.0" + ]; + [ + "0.0.0.0" + ]; + [ + "0.0.0.0" + ] ] ] } diff --git a/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.in b/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.in index 8e7fa7925852..4aa205991410 100644 --- a/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.in +++ b/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.in @@ -6,3 +6,4 @@ {"key"="fe80::215:b2ff:fea9:67ce";"subkey"="";"value"=""}; {"key"="::ffff:77.75.155.3";"subkey"="";"value"=""}; {"key"="sdfsdfsdf";"subkey"="";"value"=""}; +{"key"="0.0.0.0";"subkey"="";value=""}; \ No newline at end of file diff --git a/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.sql b/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.sql index effb59e5ee7f..1b875bc7313a 100644 --- a/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.sql +++ b/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.sql @@ -8,7 +8,9 @@ SELECT Ip::ToString(Ip::ConvertToIPv6(internal_representation)) AS all_ipv6, Ip::ToString(Ip::GetSubnet(internal_representation)) AS default_subnet, Ip::ToString(Ip::GetSubnet(internal_representation, 125)) AS small_subnet, - Ip::ToString(Ip::GetSubnet(internal_representation, 16)) AS large_subnet + Ip::ToString(Ip::GetSubnet(internal_representation, 16)) AS large_subnet, + Ip::ToString(Ip::GetSubnet(internal_representation, 32)) AS single_subnet4, + Ip::ToString(Ip::GetSubnet(internal_representation, 128)) AS single_subnet6 FROM ( SELECT Ip::FromString(key) AS internal_representation FROM Input ); From 27daa3c9418c4748d05a2cecab0c1367f8cda4bb Mon Sep 17 00:00:00 2001 From: Vadim Averin Date: Wed, 17 Jan 2024 13:07:18 +0000 Subject: [PATCH 7/9] More refactoring --- ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h b/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h index 203967cea53d..210a01694f37 100644 --- a/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h +++ b/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h @@ -78,7 +78,7 @@ namespace { SIMPLE_STRICT_UDF(TFromString, TOptionalString(TAutoMapString)) { TIpv6Address addr = TIpv6Address::FromString(args[0].AsStringRef()); auto res = SerializeAddress(addr); - return res ? valueBuilder->NewString(res.GetRef()) : TUnboxedValue(TUnboxedValuePod()); + return res ? valueBuilder->NewString(res.GetRef()) : TUnboxedValue(); } SIMPLE_UDF(TToString, char*(TAutoMapString)) { @@ -122,7 +122,8 @@ namespace { if (ref.Size() == 16) { return valueBuilder->NewString(ref); } else if (ref.Size() == 4) { - auto addr6 = TIpv6Address::FromString("::ffff:" + DeserializeAddress(ref).ToString(false)); + TIpv6Address addr4 = DeserializeAddress(ref); + auto addr6 = TIpv6Address(ui128(addr4) | ui128(0xFFFF) << 32, TIpv6Address::Ipv6); return valueBuilder->NewString(SerializeAddress(addr6).GetRef()); } else { ythrow yexception() << "Incorrect size of input, expected " @@ -152,8 +153,7 @@ namespace { ythrow yexception() << "Incorrect size of input, expected " << "4 or 16, got " << ref.Size(); } - ui128 mask = ui128(-1) << int((addr.Type() == TIpv6Address::Ipv6 ? 128 : 32) - subnetSize); - TIpv6Address beg(ui128(addr) & mask, addr.Type()); + TIpv6Address beg = LowerBoundForPrefix(addr, subnetSize); return valueBuilder->NewString(SerializeAddress(beg).GetRef()); } From ba6dbadd76e4ef3c180426dd4e17d75b82000189 Mon Sep 17 00:00:00 2001 From: Vadim Averin Date: Wed, 17 Jan 2024 13:35:24 +0000 Subject: [PATCH 8/9] Final cleanup --- .../yql/udfs/common/ip_base/lib/ip_base_udf.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h b/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h index 210a01694f37..4349cff216f2 100644 --- a/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h +++ b/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h @@ -46,7 +46,8 @@ namespace { return addr; } - TMaybe SerializeAddress(const TIpv6Address& addr) { + TString SerializeAddress(const TIpv6Address& addr) { + Y_ENSURE(addr.Type() == TIpv6Address::Ipv4 || addr.Type() == TIpv6Address::Ipv6); TString res; ui128 x = addr; if (addr.Type() == TIpv6Address::Ipv4) { @@ -69,16 +70,16 @@ namespace { ui8(x >> 8 & 0xff), ui8(x & 0xff) }; res = TString(reinterpret_cast(&addr6), sizeof addr6); - } else { - return Nothing(); } return res; } SIMPLE_STRICT_UDF(TFromString, TOptionalString(TAutoMapString)) { TIpv6Address addr = TIpv6Address::FromString(args[0].AsStringRef()); - auto res = SerializeAddress(addr); - return res ? valueBuilder->NewString(res.GetRef()) : TUnboxedValue(); + if (addr.Type() != TIpv6Address::Ipv4 && addr.Type() != TIpv6Address::Ipv6) { + return TUnboxedValue(); + } + return valueBuilder->NewString(SerializeAddress(addr)); } SIMPLE_UDF(TToString, char*(TAutoMapString)) { @@ -124,7 +125,7 @@ namespace { } else if (ref.Size() == 4) { TIpv6Address addr4 = DeserializeAddress(ref); auto addr6 = TIpv6Address(ui128(addr4) | ui128(0xFFFF) << 32, TIpv6Address::Ipv6); - return valueBuilder->NewString(SerializeAddress(addr6).GetRef()); + return valueBuilder->NewString(SerializeAddress(addr6)); } else { ythrow yexception() << "Incorrect size of input, expected " << "4 or 16, got " << ref.Size(); @@ -154,7 +155,7 @@ namespace { << "4 or 16, got " << ref.Size(); } TIpv6Address beg = LowerBoundForPrefix(addr, subnetSize); - return valueBuilder->NewString(SerializeAddress(beg).GetRef()); + return valueBuilder->NewString(SerializeAddress(beg)); } #define EXPORTED_IP_BASE_UDF \ From 24ca2e62ce9de73ac677383425625eb0d5c6cb7d Mon Sep 17 00:00:00 2001 From: Vadim Averin Date: Wed, 17 Jan 2024 16:13:33 +0000 Subject: [PATCH 9/9] Add explicit dependencies on library/cpp/ipv6_address --- ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h | 1 + ydb/library/yql/udfs/common/ip_base/lib/ya.make | 1 + 2 files changed, 2 insertions(+) diff --git a/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h b/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h index 4349cff216f2..458d4ea57c4e 100644 --- a/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h +++ b/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h @@ -2,6 +2,7 @@ #include +#include #include #include diff --git a/ydb/library/yql/udfs/common/ip_base/lib/ya.make b/ydb/library/yql/udfs/common/ip_base/lib/ya.make index 1fc0c6b9a51d..c4769eb5fdbb 100644 --- a/ydb/library/yql/udfs/common/ip_base/lib/ya.make +++ b/ydb/library/yql/udfs/common/ip_base/lib/ya.make @@ -13,6 +13,7 @@ SRCS( PEERDIR( ydb/library/yql/public/udf library/cpp/ipmath + library/cpp/ipv6_address ) END()