bitmap filtering

Signed-off-by: bowenlan-amzn <bowenlan23@gmail.com>
opensearch-project · Aug 2, 2024 · 3b0035b · 3b0035b
1 parent 0cd002d
commit 3b0035b
Show file tree

Hide file tree

Showing 12 changed files with 790 additions and 14 deletions.
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_bitmap_filtering.yml
@@ -0,0 +1,184 @@
+---
+setup:
+  - skip:
+      version: " - 2.16.99"
+      reason: bitmap filtering was added in 2.17
+  - do:
+      indices.create:
+        index: students
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            properties:
+              student_id:
+                type: integer
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - { "index": { "_index": "students", "_id": "1" } }
+          - { "name": "Jane Doe", "student_id": 111 }
+          - { "index": { "_index": "students", "_id": "2" } }
+          - { "name": "Mary Major", "student_id": 222 }
+          - { "index": { "_index": "students", "_id": "3" } }
+          - { "name": "John Doe", "student_id": 333 }
+  - do:
+      indices.create:
+        index: classes
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            properties:
+              enrolled:
+                type: binary
+                store: true
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - { "index": { "_index": "classes", "_id": "101" } }
+          -  { "enrolled": "OjAAAAEAAAAAAAEAEAAAAG8A3gA=" } # 111,222
+          -  { "index": { "_index": "classes", "_id": "102" } }
+          -  { "enrolled": "OjAAAAEAAAAAAAAAEAAAAG8A" }     # 111
+          -  { "index": { "_index": "classes", "_id": "103" } }
+          -  { "enrolled": "OjAAAAEAAAAAAAAAEAAAAE0B" }     # 333
+          -  { "index": { "_index": "classes", "_id": "104" } }
+          -  { "enrolled": "OjAAAAEAAAAAAAEAEAAAAN4ATQE=" } # 222,333
+  - do:
+      cluster.health:
+        wait_for_status: green
+
+---
+"Terms lookup on a binary field with bitmap":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: students
+        body: {
+          "query": {
+            "terms": {
+              "student_id": {
+                "index": "classes",
+                "id": "101",
+                "path": "enrolled",
+                "store": true
+              },
+              "value_type": "bitmap"
+            }
+          }
+        }
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._source.name: Jane Doe }
+  - match: { hits.hits.0._source.student_id: 111 }
+  - match: { hits.hits.1._source.name: Mary Major }
+  - match: { hits.hits.1._source.student_id: 222 }
+
+---
+"Terms query accepting bitmap as value":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: students
+        body: {
+          "query": {
+            "terms": {
+              "student_id": ["OjAAAAEAAAAAAAEAEAAAAG8A3gA="],
+              "value_type": "bitmap"
+            }
+          }
+        }
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._source.name: Jane Doe }
+  - match: { hits.hits.0._source.student_id: 111 }
+  - match: { hits.hits.1._source.name: Mary Major }
+  - match: { hits.hits.1._source.student_id: 222 }
+
+---
+"Boolean must bitmap filtering":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: students
+        body: {
+          "query": {
+            "bool": {
+              "must": [
+                {
+                  "terms": {
+                    "student_id": {
+                      "index": "classes",
+                      "id": "101",
+                      "path": "enrolled",
+                      "store": true
+                    },
+                    "value_type": "bitmap"
+                  }
+                }
+              ],
+              "must_not": [
+                {
+                  "terms": {
+                    "student_id": {
+                      "index": "classes",
+                      "id": "104",
+                      "path": "enrolled",
+                      "store": true
+                    },
+                    "value_type": "bitmap"
+                  }
+                }
+              ]
+            }
+          }
+        }
+  - match: { hits.total: 1 }
+  - match: { hits.hits.0._source.name: Jane Doe }
+  - match: { hits.hits.0._source.student_id: 111 }
+
+---
+"Boolean should bitmap filtering":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: students
+        body: {
+          "query": {
+            "bool": {
+              "should": [
+                {
+                  "terms": {
+                    "student_id": {
+                      "index": "classes",
+                      "id": "101",
+                      "path": "enrolled",
+                      "store": true
+                    },
+                    "value_type": "bitmap"
+                  }
+                },
+                {
+                  "terms": {
+                    "student_id": {
+                      "index": "classes",
+                      "id": "104",
+                      "path": "enrolled",
+                      "store": true
+                    },
+                    "value_type": "bitmap"
+                  }
+                }
+              ]
+            }
+          }
+        }
+  - match: { hits.total: 3 }
+  - match: { hits.hits.0._source.name: Mary Major }
+  - match: { hits.hits.0._source.student_id: 222 }
+  - match: { hits.hits.1._source.name: Jane Doe }
+  - match: { hits.hits.1._source.student_id: 111 }
+  - match: { hits.hits.2._source.name: John Doe }
+  - match: { hits.hits.2._source.student_id: 333 }
diff --git a/server/build.gradle b/server/build.gradle
@@ -125,6 +125,9 @@ dependencies {
   api "com.google.protobuf:protobuf-java:${versions.protobuf}"
   api "jakarta.annotation:jakarta.annotation-api:${versions.jakarta_annotation}"
 
+  // https://mvnrepository.com/artifact/org.roaringbitmap/RoaringBitmap
+  implementation 'org.roaringbitmap:RoaringBitmap:1.1.0'
+
   testImplementation(project(":test:framework")) {
     // tests use the locally compiled version of server
     exclude group: 'org.opensearch', module: 'server'

diff --git a/server/licenses/RoaringBitmap-1.1.0.jar.sha1 b/server/licenses/RoaringBitmap-1.1.0.jar.sha1
@@ -0,0 +1 @@
+9607213861158ae7060234d93ee9c9cb19f494d1