diff --git a/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h b/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h index 95c98006974f..458d4ea57c4e 100644 --- a/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h +++ b/ydb/library/yql/udfs/common/ip_base/lib/ip_base_udf.h @@ -2,7 +2,8 @@ #include -#include +#include +#include #include namespace { @@ -13,39 +14,77 @@ namespace { using TUnboxedValue = NKikimr::NUdf::TUnboxedValue; using TUnboxedValuePod = NKikimr::NUdf::TUnboxedValuePod; - struct TSerializeIpVisitor { - TStringRef operator()(const TIp4& ip) const { - return TStringRef(reinterpret_cast(&ip), 4); + struct TRawIp4 { + ui8 a, b, c, d; + }; + + struct TRawIp6 { + ui8 a1, a0, b1, b0, c1, c0, d1, d0, e1, e0, f1, f0, g1, g0, h1, h0; + }; + + TIpv6Address DeserializeAddress(const TStringRef& str) { + TIpv6Address addr; + if (str.Size() == 4) { + TRawIp4 addr4; + memcpy(&addr4, str.Data(), sizeof addr4); + addr = {addr4.a, addr4.b, addr4.c, addr4.d}; + } else if (str.Size() == 16) { + TRawIp6 addr6; + memcpy(&addr6, str.Data(), sizeof addr6); + addr = {ui16(ui32(addr6.a1) << ui32(8) | ui32(addr6.a0)), + ui16(ui32(addr6.b1) << ui32(8) | ui32(addr6.b0)), + ui16(ui32(addr6.c1) << ui32(8) | ui32(addr6.c0)), + ui16(ui32(addr6.d1) << ui32(8) | ui32(addr6.d0)), + ui16(ui32(addr6.e1) << ui32(8) | ui32(addr6.e0)), + ui16(ui32(addr6.f1) << ui32(8) | ui32(addr6.f0)), + ui16(ui32(addr6.g1) << ui32(8) | ui32(addr6.g0)), + ui16(ui32(addr6.h1) << ui32(8) | ui32(addr6.h0)), + }; + } else { + ythrow yexception() << "Incorrect size of input, expected " + << "4 or 16, got " << str.Size(); } - TStringRef operator()(const TIp6& ip) const { - return TStringRef(reinterpret_cast(&ip.Data), 16); + return addr; + } + + TString SerializeAddress(const TIpv6Address& addr) { + Y_ENSURE(addr.Type() == TIpv6Address::Ipv4 || addr.Type() == TIpv6Address::Ipv6); + TString res; + ui128 x = addr; + if (addr.Type() == TIpv6Address::Ipv4) { + TRawIp4 addr4 { + ui8(x >> 24 & 0xff), + ui8(x >> 16 & 0xff), + ui8(x >> 8 & 0xff), + ui8(x & 0xff) + }; + res = TString(reinterpret_cast(&addr4), sizeof addr4); + } else if (addr.Type() == TIpv6Address::Ipv6) { + TRawIp6 addr6 { + ui8(x >> 120 & 0xff), ui8(x >> 112 & 0xff), + ui8(x >> 104 & 0xff), ui8(x >> 96 & 0xff), + ui8(x >> 88 & 0xff), ui8(x >> 80 & 0xff), + ui8(x >> 72 & 0xff), ui8(x >> 64 & 0xff), + ui8(x >> 56 & 0xff), ui8(x >> 48 & 0xff), + ui8(x >> 40 & 0xff), ui8(x >> 32 & 0xff), + ui8(x >> 24 & 0xff), ui8(x >> 16 & 0xff), + ui8(x >> 8 & 0xff), ui8(x & 0xff) + }; + res = TString(reinterpret_cast(&addr6), sizeof addr6); } - }; + return res; + } SIMPLE_STRICT_UDF(TFromString, TOptionalString(TAutoMapString)) { - try { - TString input(args[0].AsStringRef()); - const TIp4Or6& ip = Ip4Or6FromString(input.c_str()); - return valueBuilder->NewString(std::visit(TSerializeIpVisitor(), ip)); - } catch (TSystemError&) { + TIpv6Address addr = TIpv6Address::FromString(args[0].AsStringRef()); + if (addr.Type() != TIpv6Address::Ipv4 && addr.Type() != TIpv6Address::Ipv6) { return TUnboxedValue(); } + return valueBuilder->NewString(SerializeAddress(addr)); } SIMPLE_UDF(TToString, char*(TAutoMapString)) { - const auto& ref = args[0].AsStringRef(); - if (ref.Size() == 4) { - TIp4 ip; - memcpy(&ip, ref.Data(), sizeof(ip)); - return valueBuilder->NewString(Ip4Or6ToString(ip)); - } else if (ref.Size() == 16) { - TIp6 ip; - memcpy(&ip.Data, ref.Data(), sizeof(ip.Data)); - return valueBuilder->NewString(Ip4Or6ToString(ip)); - } else { - ythrow yexception() << "Incorrect size of input, expected " - << "4 or 16, got " << ref.Size(); - } + return valueBuilder->NewString(DeserializeAddress(args[0].AsStringRef()).ToString(false)); } SIMPLE_STRICT_UDF(TIsIPv4, bool(TOptionalString)) { @@ -73,15 +112,8 @@ namespace { bool result = false; if (args[0]) { const auto ref = args[0].AsStringRef(); - if (ref.Size() == 16 && ref.Data()[10] == -1) { - bool allZeroes = true; - for (int i = 0; i < 10; ++i) { - if (ref.Data()[i] != 0) { - allZeroes = false; - break; - } - } - result = allZeroes; + if (ref.Size() == 16) { + result = DeserializeAddress(ref).Isv4MappedTov6(); } } return TUnboxedValuePod(result); @@ -92,10 +124,9 @@ namespace { if (ref.Size() == 16) { return valueBuilder->NewString(ref); } else if (ref.Size() == 4) { - TIp4 ipv4; - memcpy(&ipv4, ref.Data(), sizeof(ipv4)); - const TIp6 ipv6 = Ip6FromIp4(ipv4); - return valueBuilder->NewString(TStringRef(reinterpret_cast(&ipv6.Data), 16)); + TIpv6Address addr4 = DeserializeAddress(ref); + auto addr6 = TIpv6Address(ui128(addr4) | ui128(0xFFFF) << 32, TIpv6Address::Ipv6); + return valueBuilder->NewString(SerializeAddress(addr6)); } else { ythrow yexception() << "Incorrect size of input, expected " << "4 or 16, got " << ref.Size(); @@ -105,33 +136,27 @@ namespace { SIMPLE_UDF_WITH_OPTIONAL_ARGS(TGetSubnet, char*(TAutoMapString, TOptionalByte), 1) { const auto ref = args[0].AsStringRef(); ui8 subnetSize = args[1].GetOrDefault(0); - + TIpv6Address addr = DeserializeAddress(ref); if (ref.Size() == 4) { if (!subnetSize) { subnetSize = 24; } + if (subnetSize > 32) { + subnetSize = 32; + } } else if (ref.Size() == 16) { if (!subnetSize) { subnetSize = 64; } + if (subnetSize > 128) { + subnetSize = 128; + } } else { ythrow yexception() << "Incorrect size of input, expected " << "4 or 16, got " << ref.Size(); } - TBuffer result(ref.Data(), ref.Size()); - int bytesToMask = ref.Size() * 8 - subnetSize; - ui8 currentByte = ref.Size() - 1; - while (bytesToMask > 0) { - if (bytesToMask > 8) { - result.Data()[currentByte] = 0; - } else { - result.Data()[currentByte] = result.Data()[currentByte] & (0xff << bytesToMask); - } - bytesToMask -= 8; - --currentByte; - } - - return valueBuilder->NewString(TStringRef(result.Data(), result.Size())); + TIpv6Address beg = LowerBoundForPrefix(addr, subnetSize); + return valueBuilder->NewString(SerializeAddress(beg)); } #define EXPORTED_IP_BASE_UDF \ diff --git a/ydb/library/yql/udfs/common/ip_base/lib/ya.make b/ydb/library/yql/udfs/common/ip_base/lib/ya.make index 2d532234dd62..c4769eb5fdbb 100644 --- a/ydb/library/yql/udfs/common/ip_base/lib/ya.make +++ b/ydb/library/yql/udfs/common/ip_base/lib/ya.make @@ -12,6 +12,8 @@ SRCS( PEERDIR( ydb/library/yql/public/udf + library/cpp/ipmath + library/cpp/ipv6_address ) END() diff --git a/ydb/library/yql/udfs/common/ip_base/test/canondata/test.test_Basic_/results.txt b/ydb/library/yql/udfs/common/ip_base/test/canondata/test.test_Basic_/results.txt index c80bc2d49858..c62c9cbd35f5 100644 --- a/ydb/library/yql/udfs/common/ip_base/test/canondata/test.test_Basic_/results.txt +++ b/ydb/library/yql/udfs/common/ip_base/test/canondata/test.test_Basic_/results.txt @@ -87,6 +87,26 @@ "String" ] ] + ]; + [ + "single_subnet4"; + [ + "OptionalType"; + [ + "DataType"; + "String" + ] + ] + ]; + [ + "single_subnet6"; + [ + "OptionalType"; + [ + "DataType"; + "String" + ] + ] ] ] ] @@ -113,6 +133,12 @@ ]; [ "127.0.0.0" + ]; + [ + "127.0.0.1" + ]; + [ + "127.0.0.1" ] ]; [ @@ -136,6 +162,12 @@ ]; [ "::" + ]; + [ + "::" + ]; + [ + "::1" ] ]; [ @@ -161,6 +193,12 @@ ]; [ "213.180.0.0" + ]; + [ + "213.180.193.3" + ]; + [ + "213.180.193.3" ] ]; [ @@ -186,6 +224,12 @@ ]; [ "2a02::" + ]; + [ + "2a02:6b8::" + ]; + [ + "2a02:6b8::3" ] ]; [ @@ -211,6 +255,12 @@ ]; [ "2400::" + ]; + [ + "2400:cb00::" + ]; + [ + "2400:cb00:2048:1::681c:1b65" ] ]; [ @@ -236,6 +286,12 @@ ]; [ "fe80::" + ]; + [ + "fe80::" + ]; + [ + "fe80::215:b2ff:fea9:67ce" ] ]; [ @@ -261,6 +317,12 @@ ]; [ "::" + ]; + [ + "::" + ]; + [ + "::ffff:77.75.155.3" ] ]; [ @@ -272,7 +334,38 @@ #; #; #; + #; + #; # + ]; + [ + [ + "\0\0\0\0" + ]; + [ + "0.0.0.0" + ]; + %true; + %false; + %false; + [ + "::ffff:0.0.0.0" + ]; + [ + "0.0.0.0" + ]; + [ + "0.0.0.0" + ]; + [ + "0.0.0.0" + ]; + [ + "0.0.0.0" + ]; + [ + "0.0.0.0" + ] ] ] } diff --git a/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.in b/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.in index 8e7fa7925852..4aa205991410 100644 --- a/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.in +++ b/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.in @@ -6,3 +6,4 @@ {"key"="fe80::215:b2ff:fea9:67ce";"subkey"="";"value"=""}; {"key"="::ffff:77.75.155.3";"subkey"="";"value"=""}; {"key"="sdfsdfsdf";"subkey"="";"value"=""}; +{"key"="0.0.0.0";"subkey"="";value=""}; \ No newline at end of file diff --git a/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.sql b/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.sql index effb59e5ee7f..1b875bc7313a 100644 --- a/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.sql +++ b/ydb/library/yql/udfs/common/ip_base/test/cases/Basic.sql @@ -8,7 +8,9 @@ SELECT Ip::ToString(Ip::ConvertToIPv6(internal_representation)) AS all_ipv6, Ip::ToString(Ip::GetSubnet(internal_representation)) AS default_subnet, Ip::ToString(Ip::GetSubnet(internal_representation, 125)) AS small_subnet, - Ip::ToString(Ip::GetSubnet(internal_representation, 16)) AS large_subnet + Ip::ToString(Ip::GetSubnet(internal_representation, 16)) AS large_subnet, + Ip::ToString(Ip::GetSubnet(internal_representation, 32)) AS single_subnet4, + Ip::ToString(Ip::GetSubnet(internal_representation, 128)) AS single_subnet6 FROM ( SELECT Ip::FromString(key) AS internal_representation FROM Input );