From ad1df9e7b765d2e777cf77dec129a6cd8f11af8a Mon Sep 17 00:00:00 2001 From: "Samuel.G" <1148690954@qq.com> Date: Wed, 4 Sep 2024 20:57:09 +0800 Subject: [PATCH 1/2] fix wildcard term & terms query & add YAML REST test (#15607) * fix wildcard term & terms query & add YAML REST test Signed-off-by: gesong.samuel * add CHANGELOG Signed-off-by: gesong.samuel --------- Signed-off-by: gesong.samuel Co-authored-by: gesong.samuel --- CHANGELOG.md | 1 + .../test/search/270_wildcard_fieldtype_queries.yml | 10 ++++++++++ .../opensearch/index/mapper/WildcardFieldMapper.java | 4 ++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f67e9a4c257f7..71868de93812e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,6 +106,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Fix split response processor not included in allowlist ([#15393](https://github.com/opensearch-project/OpenSearch/pull/15393)) - Fix unchecked cast in dynamic action map getter ([#15394](https://github.com/opensearch-project/OpenSearch/pull/15394)) - Fix null values indexed as "null" strings in flat_object field ([#14069](https://github.com/opensearch-project/OpenSearch/pull/14069)) +- Fix terms query on wildcard field returns nothing ([#15607](https://github.com/opensearch-project/OpenSearch/pull/15607)) ### Security diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml index 05b6b2e5ed712..c449a06cf633d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml @@ -227,3 +227,13 @@ setup: my_field: value: ".*" - match: { hits.total.value: 5 } +--- +"terms query on wildcard field matches": + - do: + search: + index: test + body: + query: + terms: { my_field: ["AbCd"] } + - match: { hits.total.value: 1 } + - match: { hits.hits.0._id: "5" } diff --git a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java index 4998a822917b4..cd95e320209ee 100644 --- a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java @@ -635,7 +635,7 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower @Override public Query termQueryCaseInsensitive(Object value, QueryShardContext context) { - return wildcardQuery(value.toString(), MultiTermQuery.CONSTANT_SCORE_REWRITE, true, context); + return wildcardQuery(BytesRefs.toString(value), MultiTermQuery.CONSTANT_SCORE_REWRITE, true, context); } @Override @@ -649,7 +649,7 @@ public Query termsQuery(List values, QueryShardContext context) { Set expectedValues = new HashSet<>(); StringBuilder pattern = new StringBuilder(); for (Object value : values) { - String stringVal = value.toString(); + String stringVal = BytesRefs.toString(value); builder.add(matchAllTermsQuery(name(), getRequiredNGrams(stringVal)), BooleanClause.Occur.SHOULD); expectedValues.add(stringVal); if (pattern.length() > 0) { From 4f57d6aab21594a5ac20d26abbbba959aba1ab75 Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Wed, 4 Sep 2024 05:58:28 -0700 Subject: [PATCH 2/2] Fix approximate pointrangequery tests (#15590) Signed-off-by: Harsha Vamsi Kalluri --- .../ApproximatePointRangeQuery.java | 15 ++++++++++----- .../ApproximatePointRangeQueryTests.java | 19 +++++++++++-------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java b/server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java index cee8bc43d7ffd..8076da6ab970b 100644 --- a/server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java +++ b/server/src/main/java/org/opensearch/search/approximate/ApproximatePointRangeQuery.java @@ -146,10 +146,11 @@ public void grow(int count) { public void visit(int docID) { // it is possible that size < 1024 and docCount < size but we will continue to count through all the 1024 docs // and collect less, but it won't hurt performance - if (docCount[0] < size) { - adder.add(docID); - docCount[0]++; + if (docCount[0] >= size) { + return; } + adder.add(docID); + docCount[0]++; } @Override @@ -231,7 +232,7 @@ private void intersectRight(PointValues.PointTree pointTree, PointValues.Interse public void intersectLeft(PointValues.IntersectVisitor visitor, PointValues.PointTree pointTree, long[] docCount) throws IOException { PointValues.Relation r = visitor.compare(pointTree.getMinPackedValue(), pointTree.getMaxPackedValue()); - if (docCount[0] > size) { + if (docCount[0] >= size) { return; } switch (r) { @@ -279,7 +280,7 @@ public void intersectLeft(PointValues.IntersectVisitor visitor, PointValues.Poin public void intersectRight(PointValues.IntersectVisitor visitor, PointValues.PointTree pointTree, long[] docCount) throws IOException { PointValues.Relation r = visitor.compare(pointTree.getMinPackedValue(), pointTree.getMaxPackedValue()); - if (docCount[0] > size) { + if (docCount[0] >= size) { return; } switch (r) { @@ -435,6 +436,10 @@ public boolean canApproximate(SearchContext context) { if (!(context.query() instanceof ApproximateIndexOrDocValuesQuery)) { return false; } + // size 0 could be set for caching + if (context.from() + context.size() == 0) { + this.setSize(10_000); + } this.setSize(Math.max(context.from() + context.size(), context.trackTotalHitsUpTo())); if (context.request() != null && context.request().source() != null) { FieldSortBuilder primarySortField = FieldSortBuilder.getPrimaryFieldSortOrNull(context.request().source()); diff --git a/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java b/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java index 7fb75e43da6ec..1eaea6a9e9d47 100644 --- a/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java +++ b/server/src/test/java/org/opensearch/search/approximate/ApproximatePointRangeQueryTests.java @@ -127,9 +127,9 @@ public void testApproximateRangeWithSizeUnderDefault() throws IOException { } doc.add(new LongPoint("point", scratch)); iw.addDocument(doc); - if (i % 15 == 0) iw.flush(); } iw.flush(); + iw.forceMerge(1); try (IndexReader reader = iw.getReader()) { try { long lower = 0; @@ -168,6 +168,7 @@ public void testApproximateRangeWithSizeOverDefault() throws IOException { iw.addDocument(doc); } iw.flush(); + iw.forceMerge(1); try (IndexReader reader = iw.getReader()) { try { long lower = 0; @@ -185,7 +186,7 @@ protected String toString(int dimension, byte[] value) { }; IndexSearcher searcher = new IndexSearcher(reader); TopDocs topDocs = searcher.search(approximateQuery, 11000); - assertEquals(topDocs.totalHits, new TotalHits(11001, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO)); + assertEquals(topDocs.totalHits, new TotalHits(11000, TotalHits.Relation.EQUAL_TO)); } catch (IOException e) { throw new RuntimeException(e); } @@ -213,6 +214,7 @@ public void testApproximateRangeShortCircuit() throws IOException { if (i % 10 == 0) iw.flush(); } iw.flush(); + iw.forceMerge(1); try (IndexReader reader = iw.getReader()) { try { long lower = 0; @@ -258,6 +260,7 @@ public void testApproximateRangeShortCircuitAscSort() throws IOException { iw.addDocument(doc); } iw.flush(); + iw.forceMerge(1); try (IndexReader reader = iw.getReader()) { try { long lower = 0; @@ -284,12 +287,12 @@ protected String toString(int dimension, byte[] value) { assertNotEquals(topDocs.totalHits, topDocs1.totalHits); assertEquals(topDocs.totalHits, new TotalHits(10, TotalHits.Relation.EQUAL_TO)); assertEquals(topDocs1.totalHits, new TotalHits(21, TotalHits.Relation.EQUAL_TO)); - assertEquals(topDocs.scoreDocs[0].doc, 0); - assertEquals(topDocs.scoreDocs[1].doc, 1); - assertEquals(topDocs.scoreDocs[2].doc, 2); - assertEquals(topDocs.scoreDocs[3].doc, 3); - assertEquals(topDocs.scoreDocs[4].doc, 4); - assertEquals(topDocs.scoreDocs[5].doc, 5); + assertEquals(topDocs.scoreDocs[0].doc, topDocs1.scoreDocs[0].doc); + assertEquals(topDocs.scoreDocs[1].doc, topDocs1.scoreDocs[1].doc); + assertEquals(topDocs.scoreDocs[2].doc, topDocs1.scoreDocs[2].doc); + assertEquals(topDocs.scoreDocs[3].doc, topDocs1.scoreDocs[3].doc); + assertEquals(topDocs.scoreDocs[4].doc, topDocs1.scoreDocs[4].doc); + assertEquals(topDocs.scoreDocs[5].doc, topDocs1.scoreDocs[5].doc); } catch (IOException e) { throw new RuntimeException(e);