From cd4684262e3c0146f4c70877b8f90b9b0fddaeff Mon Sep 17 00:00:00 2001 From: lihuigang Date: Wed, 2 Dec 2020 22:40:32 +0800 Subject: [PATCH 01/12] bitmap_xor --- be/src/exprs/bitmap_function.cpp | 21 +++++++++++++++++++++ be/src/exprs/bitmap_function.h | 1 + 2 files changed, 22 insertions(+) diff --git a/be/src/exprs/bitmap_function.cpp b/be/src/exprs/bitmap_function.cpp index 76267bb0519b70..f0c0e6ff6dc0ce 100644 --- a/be/src/exprs/bitmap_function.cpp +++ b/be/src/exprs/bitmap_function.cpp @@ -477,6 +477,27 @@ StringVal BitmapFunctions::bitmap_and(FunctionContext* ctx, const StringVal& lhs return serialize(ctx, &bitmap); } +StringVal BitmapFunctions::bitmap_xor(FunctionContext* ctx, const StringVal& lhs, + const StringVal& rhs) { + if (lhs.is_null || rhs.is_null) { + return StringVal::null(); + } + BitmapValue bitmap; + if (lhs.len == 0) { + bitmap |= *reinterpret_cast(lhs.ptr); + } else { + bitmap |= BitmapValue((char*)lhs.ptr); + } + + if (rhs.len == 0) { + bitmap ^= *reinterpret_cast(rhs.ptr); + } else { + bitmap ^= BitmapValue((char*)rhs.ptr); + } + return serialize(ctx, &bitmap); +} + + StringVal BitmapFunctions::bitmap_to_string(FunctionContext* ctx, const StringVal& input) { if (input.is_null) { return StringVal::null(); diff --git a/be/src/exprs/bitmap_function.h b/be/src/exprs/bitmap_function.h index fa110cb3c637b0..21fbd89791c3c7 100644 --- a/be/src/exprs/bitmap_function.h +++ b/be/src/exprs/bitmap_function.h @@ -60,6 +60,7 @@ class BitmapFunctions { static StringVal to_bitmap(FunctionContext* ctx, const StringVal& src); static StringVal bitmap_hash(FunctionContext* ctx, const StringVal& src); static StringVal bitmap_or(FunctionContext* ctx, const StringVal& src, const StringVal& dst); + static StringVal bitmap_xor(FunctionContext* ctx, const StringVal& src, const StringVal& dst); static StringVal bitmap_and(FunctionContext* ctx, const StringVal& src, const StringVal& dst); static StringVal bitmap_to_string(FunctionContext* ctx, const StringVal& input); // Convert a comma separated string to a Bitmap From 8d633ef7a49cb9b98df36b0c2c2f18f9a11f6c3d Mon Sep 17 00:00:00 2001 From: lihuigang Date: Thu, 3 Dec 2020 14:19:34 +0800 Subject: [PATCH 02/12] bitmap_xor --- be/src/exprs/bitmap_function.cpp | 2 +- be/src/util/bitmap_value.h | 51 ++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/be/src/exprs/bitmap_function.cpp b/be/src/exprs/bitmap_function.cpp index f0c0e6ff6dc0ce..53f120d2204b05 100644 --- a/be/src/exprs/bitmap_function.cpp +++ b/be/src/exprs/bitmap_function.cpp @@ -490,7 +490,7 @@ StringVal BitmapFunctions::bitmap_xor(FunctionContext* ctx, const StringVal& lhs } if (rhs.len == 0) { - bitmap ^= *reinterpret_cast(rhs.ptr); + bitmap = *reinterpret_cast(rhs.ptr); } else { bitmap ^= BitmapValue((char*)rhs.ptr); } diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h index 9ccace7c5d21c4..7934be6a5b2cab 100644 --- a/be/src/util/bitmap_value.h +++ b/be/src/util/bitmap_value.h @@ -1087,6 +1087,57 @@ class BitmapValue { return *this; } + // Compute the intersection between the current bitmap and the provided bitmap. + // Possible type transitions are: + // SINGLE -> EMPTY + // BITMAP -> EMPTY + // BITMAP -> SINGLE + BitmapValue& operator^=(const BitmapValue& rhs) { + switch (rhs._type) { + case EMPTY: + _type = EMPTY; + _bitmap.clear(); + break; + case SINGLE: + switch (_type) { + case EMPTY: + break; + case SINGLE: + if (_sv != rhs._sv) { + _type = EMPTY; + } + break; + case BITMAP: + if (!_bitmap.contains(rhs._sv)) { + _type = EMPTY; + } else { + _type = SINGLE; + _sv = rhs._sv; + } + _bitmap.clear(); + break; + } + break; + case BITMAP: + switch (_type) { + case EMPTY: + break; + case SINGLE: + if (!rhs._bitmap.contains(_sv)) { + _type = EMPTY; + } + break; + case BITMAP: + _bitmap ^= rhs._bitmap; + _convert_to_smaller_type(); + break; + } + break; + } + return *this; + } + + // check if value x is present bool contains(uint64_t x) { switch (_type) { From 7b927c50013125d0b617ebde13a986965d9b3d23 Mon Sep 17 00:00:00 2001 From: lihuigang Date: Sat, 5 Dec 2020 22:28:34 +0800 Subject: [PATCH 03/12] bitmap_xor --- .rat-excludes | 33 ----------------------- be/src/util/bitmap_value.h | 26 +++++++++++------- gensrc/script/doris_builtins_functions.py | 2 ++ 3 files changed, 19 insertions(+), 42 deletions(-) delete mode 100644 .rat-excludes diff --git a/.rat-excludes b/.rat-excludes deleted file mode 100644 index f5891ba54663d2..00000000000000 --- a/.rat-excludes +++ /dev/null @@ -1,33 +0,0 @@ -.*json -.*rst -.*svg -.*xml -.*zip -.clang-format -.gitattributes -.gitignore -.rat-excludes -DISCLAIMER-WIP -LICENSE -NOTICE -gutil/* -manifest -patches/* -data/* -test_data/* -test.zip -test.zip.md5 -webroot/* -net_snmp_normal -spark_launcher_monitor.log -jmockit/* -status.* -env* -lru* -skiplist.h -string_search.hpp -coding.* -condition_variable.* -murmur_hash3.* -utf8_check.cpp -.markdownlintignore diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h index 7934be6a5b2cab..8f7dcc95cba1d6 100644 --- a/be/src/util/bitmap_value.h +++ b/be/src/util/bitmap_value.h @@ -171,7 +171,7 @@ class Roaring64Map { * Remove value x * */ - void remove(uint32_t x) { roarings[0].remove(x); } + void(uint32_t x) { roarings[0].remove(x); } void remove(uint64_t x) { auto roaring_iter = roarings.find(highBytes(x)); if (roaring_iter != roarings.cend()) roaring_iter->second.remove(lowBytes(x)); @@ -1006,6 +1006,7 @@ class BitmapValue { } } + // Compute the union between the current bitmap and the provided bitmap. // Possible type transitions are: // EMPTY -> SINGLE @@ -1051,6 +1052,7 @@ class BitmapValue { case SINGLE: switch (_type) { case EMPTY: + add(); break; case SINGLE: if (_sv != rhs._sv) { @@ -1095,36 +1097,42 @@ class BitmapValue { BitmapValue& operator^=(const BitmapValue& rhs) { switch (rhs._type) { case EMPTY: - _type = EMPTY; - _bitmap.clear(); break; case SINGLE: switch (_type) { case EMPTY: + add(rhs._sv) break; case SINGLE: - if (_sv != rhs._sv) { + if (_sv = rhs._sv) { _type = EMPTY; + _bitmap.clear(); + }else{ + add(rhs._sv) } break; case BITMAP: if (!_bitmap.contains(rhs._sv)) { - _type = EMPTY; + add(rhs._sv); } else { - _type = SINGLE; - _sv = rhs._sv; + _bitmap.remove(rhs._sv); } - _bitmap.clear(); break; } break; case BITMAP: switch (_type) { case EMPTY: + _bitmap = rhs._bitmap; + _type = BITMAP; break; case SINGLE: + _bitmap = rhs._bitmap; + _type = BITMAP; if (!rhs._bitmap.contains(_sv)) { - _type = EMPTY; + _bitmap.add(_sv); + }else{ + _bitmap.remove(_sv); } break; case BITMAP: diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index cb31d408ed5ffc..67742ba9ac8c98 100755 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -800,6 +800,8 @@ '_ZN5doris15BitmapFunctions12bitmap_emptyEPN9doris_udf15FunctionContextE'], [['bitmap_or'], 'BITMAP', ['BITMAP','BITMAP'], '_ZN5doris15BitmapFunctions9bitmap_orEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], + [['bitmap_xor'], 'BITMAP', ['BITMAP','BITMAP'], + '_ZN5doris15BitmapFunctions10bitmap_xorEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], [['bitmap_and'], 'BITMAP', ['BITMAP','BITMAP'], '_ZN5doris15BitmapFunctions10bitmap_andEPN9doris_udf15FunctionContextERKNS1_9StringValES6_'], [['bitmap_to_string'], 'VARCHAR', ['BITMAP'], From f02890f3a8bc843a24bda76f82e6eb883997d633 Mon Sep 17 00:00:00 2001 From: lihuigang Date: Sat, 5 Dec 2020 22:41:00 +0800 Subject: [PATCH 04/12] fix bitmap_xor bug --- be/src/util/bitmap_value.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h index 8f7dcc95cba1d6..d3db4d1c19580b 100644 --- a/be/src/util/bitmap_value.h +++ b/be/src/util/bitmap_value.h @@ -171,7 +171,7 @@ class Roaring64Map { * Remove value x * */ - void(uint32_t x) { roarings[0].remove(x); } + void remove(uint32_t x) { roarings[0].remove(x); } void remove(uint64_t x) { auto roaring_iter = roarings.find(highBytes(x)); if (roaring_iter != roarings.cend()) roaring_iter->second.remove(lowBytes(x)); @@ -1052,7 +1052,6 @@ class BitmapValue { case SINGLE: switch (_type) { case EMPTY: - add(); break; case SINGLE: if (_sv != rhs._sv) { @@ -1101,14 +1100,14 @@ class BitmapValue { case SINGLE: switch (_type) { case EMPTY: - add(rhs._sv) + add(rhs._sv); break; case SINGLE: - if (_sv = rhs._sv) { + if (_sv == rhs._sv) { _type = EMPTY; _bitmap.clear(); }else{ - add(rhs._sv) + add(rhs._sv); } break; case BITMAP: From e1d94037c8d838a2f17dd6698aed3b436a9374af Mon Sep 17 00:00:00 2001 From: lihuigang Date: Sun, 6 Dec 2020 08:37:56 +0800 Subject: [PATCH 05/12] bitmap_xor --- be/src/exprs/bitmap_function.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/be/src/exprs/bitmap_function.cpp b/be/src/exprs/bitmap_function.cpp index 53f120d2204b05..b9621686145c90 100644 --- a/be/src/exprs/bitmap_function.cpp +++ b/be/src/exprs/bitmap_function.cpp @@ -477,7 +477,7 @@ StringVal BitmapFunctions::bitmap_and(FunctionContext* ctx, const StringVal& lhs return serialize(ctx, &bitmap); } -StringVal BitmapFunctions::bitmap_xor(FunctionContext* ctx, const StringVal& lhs, +StringVal BitmapFunctions::bitmap_xor(FunctionContext* ctx, const StringVal& lhs, const StringVal& rhs) { if (lhs.is_null || rhs.is_null) { return StringVal::null(); @@ -490,7 +490,7 @@ StringVal BitmapFunctions::bitmap_xor(FunctionContext* ctx, const StringVal& lhs } if (rhs.len == 0) { - bitmap = *reinterpret_cast(rhs.ptr); + bitmap ^= *reinterpret_cast(rhs.ptr); } else { bitmap ^= BitmapValue((char*)rhs.ptr); } From bd35f7692df7039bbe5220be4deec705f96cda3b Mon Sep 17 00:00:00 2001 From: lihuigang Date: Mon, 7 Dec 2020 10:47:12 +0800 Subject: [PATCH 06/12] bitmap_xor --- be/src/util/bitmap_value.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h index d3db4d1c19580b..1b55730458a017 100644 --- a/be/src/util/bitmap_value.h +++ b/be/src/util/bitmap_value.h @@ -1088,7 +1088,7 @@ class BitmapValue { return *this; } - // Compute the intersection between the current bitmap and the provided bitmap. + // Compute the symmetric union between the current bitmap and the provided bitmap. // Possible type transitions are: // SINGLE -> EMPTY // BITMAP -> EMPTY From f1e4ebf110b66b996cf5a9948baa095d7a8c4ce2 Mon Sep 17 00:00:00 2001 From: lihuigang Date: Wed, 23 Dec 2020 11:47:54 +0800 Subject: [PATCH 07/12] add bitmap_xor function document --- .../bitmap-functions/bitmap_xor.md | 55 +++++++++++++++++++ .../bitmap-functions/bitmap_xor.md | 55 +++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_xor.md create mode 100644 docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_xor.md diff --git a/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_xor.md b/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_xor.md new file mode 100644 index 00000000000000..2734f36b616046 --- /dev/null +++ b/docs/en/sql-reference/sql-functions/bitmap-functions/bitmap_xor.md @@ -0,0 +1,55 @@ +--- +{ + "title": "bitmap_xor", + "language": "en" +} +--- + + + +# bitmap_xor +## description +### Syntax + +`BITMAP BITMAP_XOR(BITMAP lhs, BITMAP rhs)` + +Compute the symmetric union of two input bitmaps, return the new bitmap. + +## example + +``` +mysql> select bitmap_count(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'))) cnt; ++------+ +| cnt | ++------+ +| 2 | ++------+ + +mysql> select bitmap_to_string(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'))); ++----------------------------------------------------------------------------------------+ +| bitmap_to_string(bitmap_xor(bitmap_from_string('2,3'), bitmap_from_string('1,2,3,4'))) | ++----------------------------------------------------------------------------------------+ +| 1,4 | ++----------------------------------------------------------------------------------------+ +``` + +## keyword + + BITMAP_XOR,BITMAP diff --git a/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_xor.md b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_xor.md new file mode 100644 index 00000000000000..3e67e4fabe2288 --- /dev/null +++ b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/bitmap_xor.md @@ -0,0 +1,55 @@ +--- +{ + "title": "bitmap_xor", + "language": "zh-CN" +} +--- + + + +# bitmap_xor +## description +### Syntax + +`BITMAP BITMAP_XOR(BITMAP lhs, BITMAP rhs)` + +计算两个输入bitmap的差集，返回新的bitmap. + +## example + +``` +mysql> select bitmap_count(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'))) cnt; ++------+ +| cnt | ++------+ +| 2 | ++------+ + +mysql> select bitmap_to_string(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'))); ++----------------------------------------------------------------------------------------+ +| bitmap_to_string(bitmap_xor(bitmap_from_string('2,3'), bitmap_from_string('1,2,3,4'))) | ++----------------------------------------------------------------------------------------+ +| 1,4 | ++----------------------------------------------------------------------------------------+ +``` + +## keyword + + BITMAP_XOR,BITMAP From a5628a1727f121a240b12dcf71c9c5ce98af9e81 Mon Sep 17 00:00:00 2001 From: lihuigang Date: Sat, 26 Dec 2020 21:39:15 +0800 Subject: [PATCH 08/12] recover .rat-excludes --- .rat-excludes | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .rat-excludes diff --git a/.rat-excludes b/.rat-excludes new file mode 100644 index 00000000000000..f5891ba54663d2 --- /dev/null +++ b/.rat-excludes @@ -0,0 +1,33 @@ +.*json +.*rst +.*svg +.*xml +.*zip +.clang-format +.gitattributes +.gitignore +.rat-excludes +DISCLAIMER-WIP +LICENSE +NOTICE +gutil/* +manifest +patches/* +data/* +test_data/* +test.zip +test.zip.md5 +webroot/* +net_snmp_normal +spark_launcher_monitor.log +jmockit/* +status.* +env* +lru* +skiplist.h +string_search.hpp +coding.* +condition_variable.* +murmur_hash3.* +utf8_check.cpp +.markdownlintignore From 7e774c40a4beca6856ba499b98b2448bf9cfd3ae Mon Sep 17 00:00:00 2001 From: lihuigang <35155212+lihuigang@users.noreply.github.com> Date: Sat, 26 Dec 2020 22:01:36 +0800 Subject: [PATCH 09/12] Update be/src/util/bitmap_value.h thanks Co-authored-by: Mingyu Chen --- be/src/util/bitmap_value.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h index 1b55730458a017..9cd8a2a7c7b205 100644 --- a/be/src/util/bitmap_value.h +++ b/be/src/util/bitmap_value.h @@ -1106,7 +1106,7 @@ class BitmapValue { if (_sv == rhs._sv) { _type = EMPTY; _bitmap.clear(); - }else{ + } else { add(rhs._sv); } break; From 4a0bf8f11cb028d6cda4e7917b11d60ad218230d Mon Sep 17 00:00:00 2001 From: lihuigang <35155212+lihuigang@users.noreply.github.com> Date: Sat, 26 Dec 2020 22:02:08 +0800 Subject: [PATCH 10/12] Update be/src/exprs/bitmap_function.cpp thanks Co-authored-by: Mingyu Chen --- be/src/exprs/bitmap_function.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/exprs/bitmap_function.cpp b/be/src/exprs/bitmap_function.cpp index b9621686145c90..f0c0e6ff6dc0ce 100644 --- a/be/src/exprs/bitmap_function.cpp +++ b/be/src/exprs/bitmap_function.cpp @@ -477,7 +477,7 @@ StringVal BitmapFunctions::bitmap_and(FunctionContext* ctx, const StringVal& lhs return serialize(ctx, &bitmap); } -StringVal BitmapFunctions::bitmap_xor(FunctionContext* ctx, const StringVal& lhs, +StringVal BitmapFunctions::bitmap_xor(FunctionContext* ctx, const StringVal& lhs, const StringVal& rhs) { if (lhs.is_null || rhs.is_null) { return StringVal::null(); From ce69a112f5202a8121b5fef26dfc658f073e3afe Mon Sep 17 00:00:00 2001 From: lihuigang Date: Sun, 27 Dec 2020 16:40:37 +0800 Subject: [PATCH 11/12] remove blank lines --- be/src/util/bitmap_value.h | 1 - 1 file changed, 1 deletion(-) diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h index 9cd8a2a7c7b205..3874201c22ba9f 100644 --- a/be/src/util/bitmap_value.h +++ b/be/src/util/bitmap_value.h @@ -1006,7 +1006,6 @@ class BitmapValue { } } - // Compute the union between the current bitmap and the provided bitmap. // Possible type transitions are: // EMPTY -> SINGLE From 15b89d687da12c51b2870b4ffd66e7a3f3179ccb Mon Sep 17 00:00:00 2001 From: lihuigang Date: Tue, 29 Dec 2020 10:58:49 +0800 Subject: [PATCH 12/12] Modify format --- be/src/util/bitmap_value.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h index 1b55730458a017..07ade92937d3d8 100644 --- a/be/src/util/bitmap_value.h +++ b/be/src/util/bitmap_value.h @@ -1106,7 +1106,7 @@ class BitmapValue { if (_sv == rhs._sv) { _type = EMPTY; _bitmap.clear(); - }else{ + } else { add(rhs._sv); } break; @@ -1130,7 +1130,7 @@ class BitmapValue { _type = BITMAP; if (!rhs._bitmap.contains(_sv)) { _bitmap.add(_sv); - }else{ + } else { _bitmap.remove(_sv); } break;